diff --git a/.dockerignore b/.dockerignore index bc16e55..82449c1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,10 @@ __pycache__ node_modules evidence .DS_Store +tests +student-tests +contracts +scripts +infra +examples +*.md diff --git a/.env.example b/.env.example index d99d26a..9233482 100644 --- a/.env.example +++ b/.env.example @@ -7,14 +7,37 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 + +# Pools and timeouts (same as lab3) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_WAIT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_POOL_MAX=20 +REDIS_TIMEOUT_MS=500 +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=10 + +# lab4: S3 client and uploads +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=5 +S3_MAX_ATTEMPTS=2 +S3_POOL_MAX=20 +UPLOAD_TIMEOUT_SECONDS=60 +UPLOAD_IDLE_TIMEOUT_SECONDS=10 +DECODE_CONCURRENCY=2 +# Cleanup of abandoned uploads (janitor service): grace period and pass interval +ATTACHMENT_GRACE_SECONDS=3600 +JANITOR_INTERVAL_SECONDS=300 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3fd8009 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.example text eol=lf +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec547..7f84287 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,14 +1,14 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Бабиченко Максим, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесён весь lab3 (API на FastAPI, PostgreSQL, сессии в Redis, HAProxy + 2 реплики, идемпотентность, migrate, свои тесты). Добавлены: SeaweedFS (`s3`) с приватным bucket и volume `s3data` (MinIO недоступен в публичных реестрах); идемпотентный bootstrap `s3-init`; отдельный ключ API с правами только на свой bucket; миграция 004 (`attachments`, `message_attachments` с составными FK по `chat_id`); синхронная загрузка PNG/JPEG с потоковыми лимитами (10 MiB, 40 Мпикс до распаковки, MIME по содержимому, проверка целостности без декодирования в память), скачивание через API с проверкой участника; `attachment_ids` в сообщениях; запись «строка-намерение -> PUT -> ready» и сервис `janitor` (очистка брошенных загрузок и объектов-сирот с grace period, dry-run, отчёт о missing). Подробности — REPORT.md. +Команды запуска из чистого clone: `Copy-Item .env.example .env; python scripts/init_secrets.py; docker compose up -d --build --wait; python student-tests/collect_evidence.py` (полностью — REPORT.md, раздел «Запуск из чистого clone»). +Результаты проверок и ссылка на REPORT.md: evidence/summary.md и evidence/*.txt; таблицы замеров — REPORT.md, раздел «Эксперименты и отказы». +Ограничения / известные ошибки: один узел (lb, postgres, redis, s3 — SPOF); повтор загрузки после потерянного ответа создаёт второе вложение (так по контракту); missing-объекты janitor только находит. См. REPORT.md, «Ограничения». +Источники и использование ИИ: библиотеки и документация перечислены в REPORT.md; при написании кода и тестов использовался ИИ-ассистент (Claude), код проверен и прогнан мной. - [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [ ] Контракт и публичные тесты не ослаблены. diff --git a/.gitignore b/.gitignore index b5a7189..0a6ee0a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/ backups/ # Runtime/build artifacts .venv/ +venv/ __pycache__/ *.py[cod] node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b96e820 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt + +COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py ./ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser + +USER appuser + +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)" + +CMD ["python", "main.py"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84..527b32b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,235 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бабиченко Максим, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран +- Commit/PR: <<ссылка на MR в Gitea после публикации>> +- Использованные источники, библиотеки и помощь ИИ: всё из lab3 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py; образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine, haproxy:3.2.25-alpine). Новое: boto3/botocore 1.43.116 (S3-клиент), Pillow 12.3.0 (проверка JPEG), образ chrislusf/seaweedfs:3.80. Документация: AWS S3 (PutObject, x-amz-checksum-sha256, presigned URLs), SeaweedFS (`weed server -s3`, S3 API, статический конфиг identities с правами `Read/Write/List:`), спецификация PNG (структура чанков, CRC, IHDR, фильтры строк, Adam7), botocore Config (timeouts, `total_max_attempts`). При написании кода и тестов использовался ИИ-ассистент (Claude); весь код проверен и прогнан мной, объяснить и изменить его могу на защите. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверял на Windows 11, все команды в PowerShell. Версии Docker, Compose и ресурсы машины — в `evidence/stack.txt`. Python нужен только для тестов (стандартная библиотека, Pillow на хосте не нужен). Перед запуском остановить стенд lab3 (`docker compose down` в его папке), иначе занят порт 8080. + +```powershell +git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab4.git +cd mtusi2026lab4 +git checkout solution/bvt2303-babichenko +$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают +Copy-Item .env.example .env +python scripts/init_secrets.py # случайные пароли, S3-ключи и secrets/s3_config.json (папка в .gitignore) +docker compose up -d --build --wait # postgres, s3 -> migrate, s3-init -> 2 реплики api, janitor -> lb +docker compose ps -a # migrate и s3-init: Exited (0), остальные healthy/running + +python student-tests/collect_evidence.py # все проверки по очереди (~8 мин), вывод в evidence/, таблицы в этот отчёт +``` + +Отдельные проверки (make на моём Windows нет, ниже то же самое без make): + +| Команда из README | То же в PowerShell | +| --- | --- | +| `make check` | `python scripts/check_contract.py` | +| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` | +| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` | +| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` | +| тесты вложений (уровень 3–4) | `python student-tests/test_attachments.py` | +| S3 вне API, права ключа, bootstrap | `python student-tests/test_s3_security.py` | +| отказы S3/БД, janitor | `python student-tests/test_consistency.py [s3\|crash\|orphan\|aborted]` | +| 10+ одновременных загрузок, память | `python student-tests/upload_memory.py` | +| ручной проход очистки | `docker compose exec janitor python janitor.py --once --dry-run` | + +После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть её: `docker start ` или `docker compose up -d --wait`. На Linux/macOS вместо `.cmd` — `scripts/restart.sh` и `scripts/stop-one.sh`. + +Bootstrap: миграции применяет одноразовый сервис `migrate`, bucket и проверку прав S3 — одноразовый сервис `s3-init` (`s3_init.py`, тот же образ, что у API); оба запускаются сами при `docker compose up`, оба идемпотентны. Повторить вручную: `docker compose run --rm migrate`, `docker compose run --rm s3-init`. Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py` (новые пароли не подходят к старым volumes, поэтому удаляются вместе). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент --HTTP--> lb: HAProxy 3.2, 127.0.0.1:8080 (единственный опубликованный порт) + | round robin, без привязки клиента к реплике + +--> api-1 --+ + +--> api-2 --+--> postgres: пользователи, чаты, сообщения, МЕТАДАННЫЕ вложений (volume pgdata) + +--> redis: сессии (AOF, volume redisdata) + +--> s3 (SeaweedFS): БАЙТЫ вложений, приватный bucket (volume s3data) +janitor: раз в 5 мин удаляет брошенные загрузки и объекты-сироты (postgres + s3) +migrate: одноразово применяет migrations/*.sql +s3-init: одноразово создаёт приватный bucket и проверяет права ключей и закрытость bucket +сети: edge (lb <-> api) и data (всё остальное; internal, без выхода наружу) +``` + +| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` на 8404 | нет | edge; `127.0.0.1:8080`, `127.0.0.1:8404/stats` | не нужны | +| API ×2 | `api` (Dockerfile, `deploy.replicas: 2`) | python:3.11.13-slim | `/health/ready` (PostgreSQL + Redis) | нет | edge + data; наружу не публикуется | `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` | +| S3 | `s3` | chrislusf/seaweedfs:3.80 (`weed server -s3`: master, volume, filer и S3 в одном процессе) | S3-порт отвечает по HTTP (`wget`) | `s3data` (данные и метаданные filer) | только data; S3 8333, UI master 9333 и filer 8888 доступны только внутри стенда, наружу не публикуются | `s3_config` (конфиг ключей и прав) | +| Bootstrap S3 | `s3-init` (образ приложения, `s3_init.py`) | python:3.11.13-slim | завершается с кодом 0 | нет | data | admin-ключ + ключ API (для проверки прав) | +| Очистка | `janitor` (тот же образ) | python:3.11.13-slim | процесс работает | нет | data | `database_url` + ограниченный ключ S3 | +| Миграции | `migrate` | python:3.11.13-slim | завершается с кодом 0 | нет | data | только `database_url` | +| База данных | `postgres` | postgres:17.11-alpine | `pg_isready` | `pgdata` | data, 5432 | `POSTGRES_PASSWORD_FILE` | +| Сессии | `redis` | redis:7.4.11-alpine | `redis-cli ping` с паролем | `redisdata` (AOF) | data, 6379 | `include /run/secrets/redis_auth` | + +Почему SeaweedFS: из трёх разрешённых вариантов MinIO отпал — его образы больше не раздаются ни с Docker Hub, ни с quay.io (проверено при сборке стенда: `pull access denied` / `unauthorized`), а закреплённый тег без доступного образа не воспроизводится. У SeaweedFS образ на Docker Hub, а права ключей задаются статическим конфигом (`-s3.config`): отдельный ключ API с правами только на свой bucket получается без отдельной утилиты администрирования. Закреплена версия 3.80, `latest` не используется. + +Что перенесено из lab3 без изменений по смыслу: аутентификация и сессии в Redis, пользователи/чаты/сообщения, идемпотентность, HAProxy с проверкой готовности и аварийным режимом, двухшаговая остановка по SIGTERM, сервис `migrate`, свои тесты lab1–3. + +Что изменено в lab4: +- Новые файлы: `storage.py` (S3-клиент), `uploads.py` (приём и проверка файла), `attachments.py` (запись в две системы), `janitor.py` (очистка/сверка), `s3_init.py` (bootstrap S3), `migrations/004_attachments.sql`. +- `main.py`: endpoint загрузки, метаданных и скачивания; `attachment_ids` в сообщениях; `db.py`: вложения и связь сообщение–вложение. +- `compose.yaml`: сервисы `s3`, `s3-init`, `janitor`, volume `s3data`, пять новых секретов (admin-ключ, ключ API, конфиг ключей SeaweedFS); `infra/haproxy.cfg`: `timeout server 30s` (см. ниже); restart-скрипты перезапускают ещё `s3` и `janitor`. +- `scripts/init_secrets.py` дописывает недостающие секреты (в том числе `s3_config.json` с правами ключей) и не трогает существующие. + +### Модель данных + +`attachments` (миграция 004): `id`, `chat_id`, `owner_id`, `filename` (только для отображения), `content_type`, `size_bytes`, `sha256`, `width`, `height`, `object_key`, `state` (`uploading` | `ready`), `created_at`, `updated_at`. `object_key` = `originals/<32 случайных hex>`: его назначает сервер, имя файла в ключ не попадает (CHECK на формат). Внешний ключ `(chat_id, owner_id) -> chat_members`: загрузить можно только в свой чат. + +`message_attachments (message_id, chat_id, attachment_id, position)`: порядок вложений в сообщении. Оба внешних ключа составные и содержат один и тот же `chat_id`: `(message_id, chat_id) -> messages`, `(attachment_id, chat_id) -> attachments`. Поэтому вложение другого чата нельзя прикрепить даже прямым INSERT в обход API. Одно вложение можно прикрепить к нескольким сообщениям (контракт: «любой участник может ссылаться на доступное вложение чата»). Ограничение на текст сообщения ослаблено до `<= 16000 байт`: пустой текст допустим с вложением, проверку «текст или вложение» делает API. + +### Загрузка: что и в каком порядке проверяется + +`POST /api/v1/chats/{chat_id}/attachments?filename=...`, тело — сырые байты одного PNG/JPEG. + +1. До чтения тела: сессия (401) -> участник чата (404) -> `filename` 1–255 символов без `/`, `\`, NUL, управляющих символов, не `.`/`..` (400) -> `Content-Type` только `image/png`/`image/jpeg` (415) -> `Content-Length` больше 10 485 760 (413). При этих ошибках тело дочитывается и выбрасывается (до 20 MiB / 10 с, ничего не сохраняя), чтобы клиент получил ответ, а не обрыв соединения. +2. Тело читается кусками: размер и SHA-256 считаются на лету, данные пишутся во временный файл (до 256 KiB в памяти, остальное на диск контейнера). Первый байт сверх 10 MiB -> 413, чтение прекращается (работает и без `Content-Length`, при `Transfer-Encoding: chunked`). Таймауты: 10 с без данных, 60 с на всё тело (`UPLOAD_IDLE_TIMEOUT_SECONDS`, `UPLOAD_TIMEOUT_SECONDS`). +3. Тип по сигнатуре (`\x89PNG...`, `FF D8 FF`), не по заголовку. Не PNG/JPEG -> 415; заявленный тип не совпадает с содержимым -> 415. +4. Размер в пикселях читается из заголовка (IHDR / SOF) и проверяется ДО распаковки: больше 40 000 000 -> 413. +5. Проверка целостности с ограниченной памятью: + - PNG — своя потоковая проверка (`uploads.check_png`): CRC каждого чанка, порядок IHDR/PLTE/IDAT/IEND, допустимые тип цвета и глубина, ничего после IEND; IDAT распаковывается zlib кусками по 64 KiB (вывод не хранится), считается точный объём данных по IHDR (с учётом Adam7) и проверяется байт фильтра каждой строки. «Zip-бомба» не раздувает память: распаковка идёт порциями и обрывается, как только данных больше, чем обещал заголовок. + - JPEG — Pillow декодирует весь поток, но в масштабе 1/8 (`draft`): 40 Мпикс -> ~0,6 Мпикс, около 2 MiB. + - Одновременно в реплике проверяется не больше `DECODE_CONCURRENCY=2` файлов (CPU). + Битый файл -> 415. Любая ошибка разбора ловится и превращается в 415, а не 500. Проверка PNG сверена с Pillow: 400 случайных корректных PNG разных типов приняты, 400 испорченных (обрезка, перевёрнутый бит) отклонены. +6. Запись (см. следующий раздел) и ответ `201 Attachment` со `status=ready`, `variants=[]`, `error_code=null`. + +Временный файл удаляется в `finally` при любом исходе: оригинал после ответа существует только в S3. У контейнеров `api` нет volumes, это проверяет `test_consistency.py`. + +Скачивание: `GET /api/v1/attachments/{id}/content` — сначала проверка участника чата в PostgreSQL (иначе 404, как у несуществующего), потом GET из S3 и потоковая отдача кусками по 64 KiB с истинным `Content-Type`, `Content-Length`, `X-Content-Type-Options: nosniff`, `Content-Disposition: inline; filename*=UTF-8''...`. Без redirect и без подписанных URL. `variant` — только `original` (иное -> 400). + +### Согласованность PostgreSQL и S3 + +Распределённой транзакции нет, поэтому порядок записи выбран так, чтобы у каждого объекта в bucket была строка-указатель в БД (`attachments.py`): + +| Шаг | Если здесь отказ | +| --- | --- | +| 1. `INSERT attachments (state='uploading')` + COMMIT | в S3 ничего нет; клиенту 503 | +| 2. `PUT` объекта в S3 с `x-amz-checksum-sha256` (S3 сам сверяет SHA-256) | откат: короткий DELETE объекта (PUT мог пройти при таймауте) и удаление строки; если S3 недоступен и откат не удался, строка `uploading` остаётся указателем для janitor; клиенту 503 | +| 3. `UPDATE state='ready'` + COMMIT | клиенту 503; объект сразу НЕ удаляется: при обрыве связи COMMIT мог пройти, удаление сломало бы готовое вложение. Строка остаётся `uploading` -> её разберёт janitor | +| 4. `201` | только после шага 3 | + +Строки `uploading` не видны через API (GET -> 404) и не прикрепляются к сообщениям (404), сообщение может ссылаться только на `ready` (проверка под `FOR SHARE` в той же транзакции, что и INSERT сообщения). + +`janitor.py` (сервис `janitor`, раз в `JANITOR_INTERVAL_SECONDS=300`, один экземпляр за счёт `pg_try_advisory_lock`): +1. Строки `uploading` старше grace period: под `FOR UPDATE SKIP LOCKED` удалить объект (DELETE идемпотентен), затем строку. Если S3 недоступен — транзакция откатывается, строка остаётся до следующего прохода. +2. Обход bucket (`originals/`, постранично): объект старше grace period без строки в БД (в любом состоянии) — сирота, удаляется (перед удалением ещё раз проверяется, что строка не появилась). Живая загрузка всегда имеет строку, потому что строка коммитится до PUT. +3. Отчёт о `ready`-строках без объекта (`missing_objects`). Их janitor не удаляет: это потеря данных, её чинят восстановлением из резервной копии, а не автоматической очисткой; скачивание такого файла отвечает 503, не 500. + +Grace period: `ATTACHMENT_GRACE_SECONDS=3600`, janitor и API отказываются стартовать, если он меньше `2 x UPLOAD_TIMEOUT_SECONDS`. Самая долгая загрузка после записи строки живёт: PUT до 2 x (1 + 5) с + откат 2 с + UPDATE до 8 с, то есть меньше 30 с — в 120 раз меньше grace period. Поэтому janitor не может удалить загрузку, которая ещё идёт, а если бы реплика зависла дольше часа, её UPDATE не найдёт строку (`state='uploading'` уже удалена под блокировкой) и клиент получит 503, а не «ready без объекта». Есть `--dry-run` (только отчёт) и `--once`. + +Обрыв загрузки клиентом: строка пишется только после приёма и проверки всего тела, поэтому оборванная загрузка не оставляет ни строки, ни объекта; временный файл удаляется сразу. + +### Готовность и отказ S3 + +S3 не входит в `/health/ready`: без него не работают только загрузка и скачивание файлов, а вход, чаты, сообщения (в том числе со ссылкой на уже загруженные вложения) и метаданные вложений продолжают работать. Если бы readiness зависел от S3, отказ S3 вывел бы из балансировки все реплики и остановил бы весь мессенджер. При недоступном S3 файловые операции отвечают `503 unavailable` без подробностей. Критичны для readiness, как и в lab3, PostgreSQL и Redis. + +S3-клиент: connect timeout 1 с, read timeout 5 с, всего 2 попытки (`total_max_attempts`), откат — 1 попытка с таймаутом 2 с. Худший ответ при зависшем S3 около 13 с, поэтому в HAProxy `timeout server` увеличен с 15 до 30 с (пока клиент передаёт тело файла, этот таймаут не срабатывает: передачу ограничивают таймауты самого API). + +### Безопасность хранилища + +- Bucket приватный: в конфиге ключей SeaweedFS нет identity `anonymous`, поэтому любой запрос без подписи получает 403. Проверка вне API: `s3-init` и `test_s3_security.py` делают анонимные GET/HEAD/LIST/PUT прямо в S3 из внутренней сети — 403. +- Два ключа (`secrets/s3_config.json`, генерирует `scripts/init_secrets.py`): `admin` (Admin/Read/Write/List) есть только у `s3` (в конфиге) и `s3-init`; `messenger-api` для api и janitor — только `Read:messenger-attachments`, `Write:messenger-attachments`, `List:messenger-attachments`. Создать/удалить bucket, поменять политику или versioning ключ API не может — проверяют `s3-init` и `test_s3_security.py`. +- Порты S3 (8333), master (9333) и filer (8888) не публикуются, SeaweedFS только во внутренней сети `data` (`internal: true`); веб-интерфейсы master/filer доступны только контейнерам стенда, с хоста — connection refused. +- Bootstrap идемпотентен: `create_bucket` при существующем bucket получает `BucketAlreadyOwnedByYou/BucketAlreadyExists` и продолжает; проверочный объект ключа API создаётся и удаляется; в конце скрипт проверяет, что bucket private. Повторный `docker compose run --rm s3-init` ничего не ломает (тест). Права ключей — декларативный конфиг, применяется при старте `s3`. +- Приватный bucket и случайный ключ не заменяют проверку доступа в API: ключ объекта не секрет (может попасть в лог, бэкап, ошибку), а bucket «приватен» для всех, у кого нет ключа S3, — но не различает пользователей мессенджера. Кто участник какого чата, знает только API, поэтому проверка участника делается на каждый GET метаданных и байтов. + +### Соответствие `.env.example` + +| Переменная | Что делает в этом решении | +| --- | --- | +| APP_HOST, APP_PORT, APP_ENV | где слушает API (0.0.0.0:8080 внутри контейнера), APP_ENV пишется в лог | +| SESSION_TTL_SECONDS, SESSION_STORE=redis | срок жизни сессии 3600 с, сессии в Redis (как в lab3) | +| DATABASE_URL_FILE, CACHE_URL_FILE | пути к секретам с адресами PostgreSQL и Redis | +| API_REPLICAS | число реплик `api`, по умолчанию 2 | +| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) | +| S3_ENDPOINT, S3_REGION, S3_BUCKET | `http://s3:8333`, `us-east-1`, `messenger-attachments` (path-style адресация) | +| S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE | ограниченный ключ API (identity `messenger-api` в `s3_config.json`) | +| MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS | 10 485 760 байт и 40 000 000 пикселей | +| S3_CONNECT_TIMEOUT_SECONDS, S3_READ_TIMEOUT_SECONDS, S3_MAX_ATTEMPTS, S3_POOL_MAX | 1 с, 5 с, 2 попытки всего, 20 соединений | +| UPLOAD_TIMEOUT_SECONDS, UPLOAD_IDLE_TIMEOUT_SECONDS | 60 с на всё тело, 10 с без данных | +| DECODE_CONCURRENCY | одновременных проверок изображений в реплике, 2 | +| ATTACHMENT_GRACE_SECONDS, JANITOR_INTERVAL_SECONDS | grace period 3600 с, проход janitor раз в 300 с | +| DB_*, REDIS_*, SHUTDOWN_* | пулы, таймауты и остановка (как в lab3) | ## Доказательства критериев +Все файлы `evidence/*.txt` создаёт `python student-tests/collect_evidence.py` (вывод тестов как есть, с датой, commit и кодом выхода); сводка — `evidence/summary.md`. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS через балансировщик, с загрузкой, сообщением с `attachment_ids`, скачиванием и 404 постороннему | evidence/smoke.txt | +| Skeleton и контракт не изменены | `python scripts/check_contract.py` | PASS; `contracts/`, `tests/`, README не менялись | evidence/check.txt | +| 3: возможности lab3 сохранены | `test_balancing.py`, `test_idempotency.py`, `test_api.py`, `test_race.py`, `test_db.py`, `redis_info.py`, `explain.py`, `test_failures.py`, публичный failover | все PASS через балансировщик | evidence/balancing.txt, idempotency.txt, student-tests.txt, race.txt, db.txt, redis.txt, explain.txt, failures.txt, failover.txt | +| 3: S3 в общем стенде; метаданные в PostgreSQL, байты в приватном bucket, не в БД и не в каталоге API | `check_stack.py`, `test_s3_security.py`, `test_consistency.py aborted` | `s3` и `s3-init` в compose; у `api` нет volumes, временных файлов после запросов 0; анонимный доступ к bucket 403 | evidence/stack.txt, s3-security.txt, consistency.txt | +| 3: POST -> 201 ready после сохранения; оригинал побайтно; message с attachment_ids читается участниками; посторонний без доступа | smoke, `test_attachments.py` (roundtrip, access, message) | SHA-256 и размер совпадают, скачивает второй участник, посторонний 404 на метаданные и байты | evidence/smoke.txt, attachments.txt | +| 3: реальные PNG/JPEG ≤10 MiB и ≤40 Мпикс; MIME по содержимому; опасные filename; лимиты до неограниченного чтения/декодирования | `test_attachments.py` | ровно 10 MiB -> 201, +1 байт -> 413 (с Content-Length и chunked); 8000x5000 -> 201, 8001x5000 -> 413, «бомба» 100000x100000 -> 413 по заголовку; 13 опасных имён -> 400 | evidence/attachments.txt | +| 3: upload -> остановка обслужившей реплики -> download через другую | публичный `make test-failover` (lab4 загружает файл до отказа и скачивает после) для каждой реплики; `test_balancing.py` | PASS для обеих реплик; файл, загруженный через одну реплику, скачан через другую | evidence/failover.txt, balancing.txt | +| 3: restart с сохранением volumes сохраняет метаданные и оригинал | `run_persistence.py` (публичный persistence + ID/время старта контейнеров) | postgres, redis, s3, обе api, janitor, lb перезапущены (те же контейнеры, новое время старта); файл скачивается с прежним токеном | evidence/persistence.txt | +| 4: тесты: слишком большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; нет 500 | `test_attachments.py` | 413; 10 видов битых файлов -> 415; 7 подмен типа -> 415; неизвестный id и вложение другого чата (автор состоит в обоих) -> 404; дубли, 11 id -> 400; ни одного 500 | evidence/attachments.txt | +| 4: прерывание загрузки и отказ между S3/БД не оставляют вечных неучтённых объектов; механизм с grace period | `test_consistency.py` (aborted, crash, orphan, s3) | обрыв: строк и объектов не прибавилось; падение после PUT: строка `uploading` невидима, janitor не трогает её до grace, `--dry-run` только отчёт, затем удалены объект и строка, повтор — 0; сирота удаляется только после grace; `ready` не трогаются; missing — только отчёт | evidence/consistency.txt | +| 4: недоступность S3 | `test_consistency.py s3` (`docker compose stop s3` и `pause s3`) | файловые операции 503 без подробностей, остальное работает; после восстановления файлы на месте, мусор убран | evidence/consistency.txt | +| 4: лимиты/таймауты и потоковая обработка; ≥10 одновременных загрузок с пиком памяти | `upload_memory.py` | таблица ниже | evidence/upload-memory.txt, upload-memory.json | +| 4: bootstrap идемпотентен | `test_s3_security.py` (повторный `docker compose run --rm s3-init`) | exit 0, «bucket is private», старые файлы скачиваются, новая загрузка 201 | evidence/s3-security.txt | +| 4: отдельные S3 credentials с ограниченными правами | `test_s3_security.py`, `check_stack.py`, `s3-init` | ключ API: get/put/delete/list своего bucket — да; create/delete bucket, policy, versioning — отказ; admin-ключ и конфиг ключей только у s3 и s3-init | evidence/s3-security.txt, stack.txt | +| 4: консоль не открыта; проверка private bucket вне API | `test_s3_security.py`, логи `s3-init` | 127.0.0.1:8333/9333/8888 — connection refused, s3 только в сети data; анонимные GET/HEAD/LIST/PUT/policy прямо в S3 -> 403 | evidence/s3-security.txt | +| Выбранный трек 5 | — | не выбран | — | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотезы: +1. Память API-реплики почти не зависит от числа и размера одновременных загрузок: тело идёт во временный файл, PNG проверяется потоково, JPEG — в масштабе 1/8, проверок одновременно не больше 2 на реплику. Наивный вариант (тело в памяти + полное декодирование 40 Мпикс ~160 MiB) при 10 одновременных загрузках не поместился бы в лимит 512 MiB. +2. Отказ S3 затрагивает только операции с байтами файлов и не оставляет видимых или вечных «половинчатых» вложений. + +Методика памяти (`student-tests/upload_memory.py`): три сценария — 10 x PNG из случайного шума ~9,5 MiB (почти несжимаемый, максимум трафика); 10 x PNG 8000x5000 RGB (40 Мпикс, максимум распаковки); 20 смешанных. Перед каждым сценарием `docker compose restart api` (сбрасывает пик), прогрев 8 загрузками, затем все потоки стартуют одновременно (barrier) через балансировщик. Пик RSS — `VmHWM` из `/proc/1/status` процесса API (максимум за жизнь процесса по данным ядра, короткие всплески не теряются), дополнительно пик памяти cgroup контейнера и опрос раз в ~0,1–0,5 с. Проверяются коды 201 и SHA-256 скачанных файлов. Клиент — тот же компьютер, поэтому время загрузки ограничено и клиентом; это замер памяти, а не пропускной способности. + + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 | +| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 | +| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 | +| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 | +| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | + + +Вывод по памяти: рост RSS реплики при 10–20 одновременных загрузках — единицы MiB (см. таблицу), а не «размер файла x число загрузок»; верхняя оценка на загрузку — 256 KiB буфера временного файла + 64 KiB куска чтения + буферы boto3 при PUT; на проверку — 64 KiB распаковки PNG или ~2 MiB для JPEG, не больше 2 одновременно. Пик cgroup выше пика RSS, потому что в него входит page cache временных файлов (ядро освобождает его при нехватке памяти, OOM он не вызывает). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается отказ одной API-реплики и временная недоступность S3 (файловые операции 503, остальное работает). Не выдерживается отказ единственных `lb`, `postgres`, `redis`, `s3` и всей машины: стенд на одном узле. +- Повтор загрузки после потерянного ответа создаёт второе вложение (в контракте нет ключа идемпотентности для загрузки). Лишнее вложение, не прикреплённое к сообщению, остаётся: удаления в API нет; janitor удаляет только незавершённые загрузки. +- `missing_objects` janitor только находит; восстановление — из резервной копии S3, которой в стенде нет. +- Обход bucket janitor’ом держит в памяти список ключей (для учебного объёма — мегабайты); для миллионов объектов нужна сверка по диапазонам ключей или S3 Inventory. +- Временный файл загрузки (часть тела сверх 256 KiB) лежит на диске контейнера. В Linux `tempfile.TemporaryFile` создаётся уже без имени в каталоге (O_TMPFILE), поэтому даже при SIGKILL реплики место освобождается ядром, «забытых» файлов не бывает. Одновременно на диске может быть до 10 MiB на каждую идущую загрузку; лимита на число одновременных загрузок, кроме лимитов HAProxy/uvicorn, нет. +- Проверяется структура изображения, а не его безопасность для всех декодеров (например, JPEG с некорректными, но декодируемыми данными пройдёт). Перекодирование оригинала — задача lab5. +- MinIO собирался быть первым выбором, но его образы больше не доступны в публичных реестрах: закреплённый тег защищает от неожиданного обновления, но не от исчезновения образа. Для эксплуатации нужно зеркалить образы в свой реестр. SeaweedFS запущен одним процессом `weed server` на одном диске — это SPOF и не защита от потери диска; веб-интерфейсы master/filer без авторизации, закрыты только сетью. +- Скрипты `failover_metrics.py`, `rolling_update.py`, `test_expiry.py` из lab3 остались в `student-tests/` и работают со стендом, но в общий прогон lab4 не входят (это уровень 4 lab3, а не lab4). diff --git a/attachments.py b/attachments.py new file mode 100644 index 0000000..cb0595b --- /dev/null +++ b/attachments.py @@ -0,0 +1,72 @@ +"""Сохранение вложения: PostgreSQL (метаданные) + S3 (байты) без распределённой транзакции. + +Порядок записи и что происходит при отказе на каждом шаге: + + 1. INSERT attachments (state=uploading) и COMMIT — упало: в S3 ничего нет, клиенту 503. + 2. PUT объекта в S3 — упало: удаляем объект (на случай, если PUT + на самом деле прошёл) и строку, клиенту 503. + Если и это не вышло — остаётся uploading-строка, + её уберёт janitor после grace period. + 3. UPDATE state=ready и COMMIT — упало/неизвестно: клиенту 503; строка остаётся + uploading, объект есть -> janitor удалит оба. + Объект сразу НЕ удаляем: при обрыве связи COMMIT + мог пройти, и удаление сломало бы ready-вложение. + 4. 201 клиенту — только после шага 3. + +Клиенту видны и прикрепляемы только ready-строки, поэтому незавершённая загрузка ни на что не влияет. +""" +import logging +from uuid import UUID, uuid4 + +import db +import storage +from errors import unavailable +from uploads import ImageInfo, Upload + +log = logging.getLogger("messenger.attachments") + + +def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict: + attachment_id = uuid4() + key = storage.new_key(uuid4().hex) + db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type, + upload.size, upload.sha256, image.width, image.height, key) + try: + storage.put(key, upload.file, upload.size, image.content_type, upload.sha256) + except storage.StorageUnavailable: + compensate(attachment_id, key) + raise unavailable("File storage is temporarily unavailable") from None + row = db.finish_upload(attachment_id) + if row is None: + # janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace). + log.warning("Upload %s was reclaimed before it finished", attachment_id) + raise unavailable() + return row + + +def compensate(attachment_id: UUID, key: str) -> None: + """Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor.""" + try: + storage.delete(key, quick=True) + db.delete_upload_intent(attachment_id) + log.info("Upload %s rolled back after S3 failure", attachment_id) + except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту + log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__) + + +def public(row: dict) -> dict: + """Строка БД -> Attachment из контракта (без object_key и размеров в пикселях).""" + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": "ready", + "variants": [], + "error_code": None, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..c21212f --- /dev/null +++ b/auth.py @@ -0,0 +1,139 @@ +import base64 +import binascii +import hashlib +import logging +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from redis.backoff import NoBackoff +from redis.retry import Retry + +import settings +from errors import BASIC_CHALLENGE, unauthorized, unavailable + +log = logging.getLogger("messenger.auth") + +hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1) + +DUMMY_HASH = hasher.hash("there-is-no-such-user") + +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def password_matches(password_hash: str | None, password: str) -> bool: + try: + hasher.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """'Basic base64(username:password)' -> (username, password) или None. + + Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617). + """ + scheme, _, value = (header or "").strip().partition(" ") + if scheme.lower() != "basic" or not value or len(value) > 1024: + return None + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + username, colon, password = decoded.partition(":") + if not colon or not username or not username.isascii(): + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + scheme, _, value = (header or "").strip().partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value: + return None + return value + + + +KEY_PREFIX = "session:" + +_redis = redis.Redis( + connection_pool=redis.BlockingConnectionPool.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + max_connections=settings.REDIS_POOL_MAX, + timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) +) + + +@dataclass(frozen=True) +class Session: + user_id: UUID + key: str + + +def session_key(token: str) -> str: + return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: UUID) -> tuple[str, datetime]: + """Новая сессия: случайный токен и абсолютное время истечения.""" + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS) + try: + _redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000)) + except redis.RedisError as exc: + log.warning("Redis is unavailable on login: %s", type(exc).__name__) + raise unavailable() from None + return token, expires_at + + +def check_bearer(header: str | None) -> Session: + token = parse_bearer(header) + if token is None: + raise unauthorized("Bearer token is required") + key = session_key(token) + try: + user_id = _redis.get(key) + except redis.RedisError as exc: + log.warning("Redis is unavailable: %s", type(exc).__name__) + raise unavailable() from None + if user_id is None: + raise unauthorized("Session is invalid or expired") + return Session(UUID(user_id), key) + + +def delete_session(session: Session) -> None: + try: + _redis.delete(session.key) + except redis.RedisError: + raise unavailable() from None + + +def ping_redis() -> None: + try: + _redis.ping() + except redis.RedisError: + raise unavailable() from None + + +def login_failed(): + return unauthorized("Invalid username or password", BASIC_CHALLENGE) + + +def close() -> None: + _redis.close() + _redis.connection_pool.disconnect() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..f718871 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,211 @@ +name: messenger-lab4 + +x-app-build: &app-build + context: . + args: + APP_VERSION: ${APP_VERSION:-dev} + +x-s3-api-secrets: &s3-api-secrets + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + migrate: + build: *app-build + command: ["python", "migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений. + # Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config + # (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket. + # Анонимного доступа нет: identity "anonymous" в конфиге не объявлена. + s3: + image: chrislusf/seaweedfs:3.80 + entrypoint: ["/usr/bin/weed"] + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0", + "-master.volumeSizeLimitMB=1024", "-volume.max=0", + "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: ["s3data:/data"] + networks: [data] + healthcheck: + # S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт») + test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 15s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки. + s3-init: + build: *app-build + command: ["python", "s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + api: + build: *app-build + env_file: .env + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3-api-secrets + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "1.00", memory: 512M} + + # Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ. + janitor: + build: *app-build + command: ["python", "janitor.py"] + env_file: .env + environment: + CACHE_URL_FILE: "" # janitor не ходит в Redis + <<: *s3-api-secrets + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + healthcheck: + disable: true + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + + lb: + image: haproxy:3.2.25-alpine + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + postgres: + image: postgres:17.11-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] + secrets: [postgres_password] + volumes: ["pgdata:/var/lib/postgresql/data"] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + redis: + image: redis:7.4.11-alpine + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + - redisdata:/data + secrets: [redis_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + +networks: + edge: {} + data: + internal: true + +volumes: + pgdata: {} + redisdata: {} + s3data: {} + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + redis_auth: {file: ./secrets/redis_auth} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + s3_config: {file: ./secrets/s3_config.json} diff --git a/db.py b/db.py new file mode 100644 index 0000000..eb11ac2 --- /dev/null +++ b/db.py @@ -0,0 +1,359 @@ +import logging +import time +from contextlib import contextmanager +from datetime import datetime +from uuid import UUID, uuid4 + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +import settings +from errors import bad_request, conflict, not_found, unavailable + +log = logging.getLogger("messenger.db") + + +pool = ConnectionPool( + conninfo=settings.DATABASE_URL, + min_size=settings.DB_POOL_MIN, + max_size=settings.DB_POOL_MAX, + timeout=settings.DB_POOL_WAIT_SECONDS, + kwargs={ + "connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS, + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" + " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", + }, + check=ConnectionPool.check_connection, + open=False, +) + + +@contextmanager +def connection(wait_seconds: float | None = None): + try: + with pool.connection(timeout=wait_seconds) as conn: + yield conn + except (psycopg.OperationalError, PoolTimeout) as exc: + log.warning("PostgreSQL is unavailable: %s", type(exc).__name__) + raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text", "filename") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} + + +def ping() -> None: + with connection(wait_seconds=1) as conn: + conn.execute("SELECT 1") + + +MIGRATIONS_LOCK = 20260202 + + +def migration_files() -> list: + return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) + + +def run_migrations() -> list[str]: + files = migration_files() + applied = [] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now())""") + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + applied.append(path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) + return applied + + +def wait_and_migrate(max_wait_seconds: int = 60) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + applied = run_migrations() + log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") + return + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + log.warning("PostgreSQL is not ready yet, retry in 1 s") + time.sleep(1) + + +def missing_migrations() -> list[str]: + expected = [path.name for path in migration_files()] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + try: + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + except pg_errors.UndefinedTable: + return expected + return [name for name in expected if name not in done] + + +def wait_for_schema(max_wait_seconds: int) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + missing = missing_migrations() + except psycopg.OperationalError: + missing = None + if missing == []: + log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) + return + if time.monotonic() > deadline: + raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") + log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None + else "missing " + ", ".join(missing)) + time.sleep(1) + + + +USER_COLUMNS = "id, username, display_name, created_at" + + +def create_user(username: str, display_name: str, password_hash: str) -> dict: + try: + with connection() as conn: + return from_db(conn.execute( + f"""INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) + except pg_errors.UniqueViolation: + raise conflict("Username already exists") from None + + +def get_user(user_id: UUID) -> dict | None: + with connection() as conn: + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) + + +def get_user_with_password(username: str) -> dict | None: + with connection() as conn: + return from_db(conn.execute( + f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) + ).fetchone()) + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" + + +def create_chat(title: str, member_ids: list[UUID]) -> dict: + chat_id = uuid4() + members = sorted(member_ids) + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, to_db(title)), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in members], + ) + except pg_errors.ForeignKeyViolation: + raise bad_request("Unknown member") from None + return {**from_db(row), "member_ids": members} + + +def get_chat(chat_id: UUID, user_id: UUID) -> dict: + with connection() as conn: + chat = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s""", + (user_id, chat_id), + ).fetchone() + if chat is None: + raise not_found("Chat not found") + return from_db(chat) + + +def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" + where, params = "", [user_id] + if after is not None: + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params += list(after) + with connection() as conn: + rows = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + {where} + ORDER BY c.created_at, c.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids""" + + +def is_member(conn, chat_id: UUID, user_id: UUID) -> bool: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() is not None + + +def check_member(chat_id: UUID, user_id: UUID) -> None: + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + + +def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID], + idempotency_key: str, request_hash: bytes) -> dict: + try: + with connection() as conn: + with conn.transaction(): + if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404 + raise not_found("Chat not found") + if attachment_ids: + # Только готовые вложения ЭТОГО чата. FOR SHARE: строку не удалят до конца транзакции. + found = conn.execute( + """SELECT id FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state = 'ready' FOR SHARE""", + (attachment_ids, chat_id), + ).fetchall() + if len(found) != len(attachment_ids): + raise not_found("Attachment not found") + created = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), + ).fetchone() + if created is not None: + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + """INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES (%s, %s, %s, %s)""", + [(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)], + ) + return {**from_db(created), "attachment_ids": list(attachment_ids)} + saved = conn.execute( + f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m + WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""", + (chat_id, sender_id, idempotency_key), + ).fetchone() + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен + raise unavailable() + if bytes(saved.pop("request_hash")) != request_hash: + raise conflict("Idempotency-Key was already used with a different request") + return from_db(saved) + + +def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """История чата после курсора (keyset-пагинация), до limit+1 строк.""" + where, params = "m.chat_id = %s", [chat_id] + if after is not None: + where += " AND (m.created_at, m.id) > (%s, %s)" + params += list(after) + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + rows = conn.execute( + f"""SELECT {MESSAGE_COLUMNS} FROM messages m + WHERE {where} + ORDER BY m.created_at, m.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + +# ---------- вложения ---------- + +ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "created_at, updated_at, object_key") + + +def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, + size_bytes: int, sha256: str, width: int, height: int, object_key: str) -> None: + """Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в S3. + + Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт + и удалит его, если загрузка не дошла до ready (обрыв, падение реплики, отказ S3 или БД). + """ + try: + with connection() as conn: + conn.execute( + """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, object_key, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""", + (attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256, + width, height, object_key), + ) + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + + +def finish_upload(attachment_id: UUID) -> dict | None: + """Шаг 3 из 3: объект в S3 уже сохранён -> ready. None, если строку успел убрать janitor.""" + with connection() as conn: + row = conn.execute( + f"""UPDATE attachments SET state = 'ready', updated_at = now() + WHERE id = %s AND state = 'uploading' + RETURNING {ATTACHMENT_COLUMNS}""", + (attachment_id,), + ).fetchone() + return from_db(row) + + +def delete_upload_intent(attachment_id: UUID) -> None: + with connection(wait_seconds=1) as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,)) + + +def get_attachment(attachment_id: UUID, user_id: UUID) -> dict: + """Готовое вложение, если user_id — участник его чата. Иначе 404 (не раскрываем существование).""" + with connection() as conn: + row = conn.execute( + f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))} + FROM attachments a + JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s + WHERE a.id = %s AND a.state = 'ready'""", + (user_id, attachment_id), + ).fetchone() + if row is None: + raise not_found("Attachment not found") + return from_db(row) diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..a641476 --- /dev/null +++ b/errors.py @@ -0,0 +1,40 @@ +# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт. +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions +BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "Service temporarily unavailable") -> ApiError: + return ApiError(503, "unavailable", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 0000000..dc0ed88 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,67 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-10-21 21:03:05, commit: 2dbed80 +# command: python.exe student-tests/test_attachments.py +# exit code: 0, 4.0 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 201 +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 201 5ffe6847 +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200 +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +11 passed, 0 failed diff --git a/evidence/balancing.txt b/evidence/balancing.txt new file mode 100644 index 0000000..dbbf431 --- /dev/null +++ b/evidence/balancing.txt @@ -0,0 +1,18 @@ +# balancing: balancing, shared sessions/data/files, X-Instance-Id +# date: 2026-10-21 21:08:50, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_balancing.py +# exit code: 0, 1.9 s + +20 x GET /users/me with one token -> replicas: {'6128eb268c44': 10, '11a93acecaca': 10} +PASS: one session works on every replica, no second login (sessions are shared in Redis) +message written by 11a93acecaca; read back by ['11a93acecaca', '6128eb268c44'] +PASS: a message written via one replica is read right away via another (shared PostgreSQL) +logout on 6128eb268c44; old token then rejected by ['11a93acecaca', '6128eb268c44'] +PASS: logout on one replica revokes the session on all of them +first POST on 6128eb268c44 -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af; same key on 11a93acecaca -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af +PASS: Idempotency-Key is shared: a retry through another replica returns the same message +file uploaded via 6128eb268c44; downloaded byte for byte via ['11a93acecaca', '6128eb268c44'] +PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local) +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['11a93acecaca', '6128eb268c44']; api containers: ['11a93acecaca', '6128eb268c44'] +PASS: X-Instance-Id is in every response and points to a real container +6 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..61c52c6 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-10-21 21:02:57, commit: 2dbed80 +# command: python.exe scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 0000000..b345844 --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,47 @@ +# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage +# date: 2026-10-21 21:08:14, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_consistency.py +# exit code: 0, 148.7 s + +declared 9 000 000 bytes, sent 211817, closed the connection +declared 9 000 000 bytes, sent 3111817, closed the connection +rows in attachments: 32 -> 32; objects in bucket: 31 -> 31 +temporary upload files in api containers: {'11a93acecaca': 0, '6128eb268c44': 0} (removed when the request ends) +api containers have no volumes: the original is never stored locally +PASS: an interrupted upload leaves no row, no object and no local file +crash after S3 PUT: row 04aacbf6 state=uploading, object originals/219175ec1d5647f0aeaec84ee8243cb3 exists in S3 +the half-written attachment is invisible: GET -> 404, attach to a message -> 404 +janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.101} -> fresh row and object are kept (the upload might still be running) +row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.092} -> nothing deleted +janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.096} -> row and object deleted +janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.103} -> idempotent +the ready attachment of the same chat is untouched +PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period +object originals/867f5c5d42bf4bf9973b60ef8856d88f put into the bucket without a row in PostgreSQL +janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.102} -> fresh object kept +after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.098} -> reported, kept +janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.1} -> orphan deleted +ready attachments older than the grace period are untouched (they have rows) +object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500) +PASS: orphans are found and deleted only after the grace period; missing objects are reported +--- S3 stopped (connection refused / name not resolved): docker compose stop s3 +upload -> 503 unavailable in 20141 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose start s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 35, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.099} +PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up +--- S3 frozen (no answer, timeouts): docker compose pause s3 +upload -> 503 unavailable in 14766 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose unpause s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 37, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.094} +PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up +5 passed, 0 failed diff --git a/evidence/db.txt b/evidence/db.txt new file mode 100644 index 0000000..481e403 --- /dev/null +++ b/evidence/db.txt @@ -0,0 +1,21 @@ +# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations +# date: 2026-10-21 21:09:10, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_db.py +# exit code: 0, 14.2 s + +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 57 -> 58 +PASS: chat row was inserted and then rolled back together with members +direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" +PASS: the database itself rejects a message from a non-member +stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97) +PASS: password is stored as Argon2id hash +unique constraint on users: UNIQUE (username) +PASS: username is unique in the database +applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; running the migrate service once more +after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; log: Migrations applied: none (schema is up to date) +PASS: migrations are applied once, a repeated run does nothing +api replicas that checked the schema at start: 8; api log lines 'Migrations applied': 0; migrate service: Exited (0) 6 minutes ago +PASS: only the migrate service applies migrations, replicas wait for the schema +unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) +PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database +7 passed, 0 failed diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 0000000..678f473 --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,99 @@ +=== Data volume + messages_total | messages_in_big_chat +----------------+---------------------- + 20146 | 15000 + + +=== History index + indexdef +---------------------------------------------------------------------------------------- + CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id) + + +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + QUERY PLAN +------------------------------------------------------------------------------------------------ + Limit (actual time=0.005..0.011 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) + Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) + Buffers: shared hit=5 + Planning: + Buffers: shared hit=6 + Planning Time: 0.060 ms + Execution Time: 0.020 ms + + +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.005..0.012 rows=51 loops=1) + Buffers: shared hit=6 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) + Buffers: shared hit=6 + Planning Time: 0.044 ms + Execution Time: 0.031 ms + + +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.020..0.225 rows=51 loops=1) + Buffers: shared hit=60 + -> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1) + Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) + Buffers: shared hit=60 + SubPlan 1 + -> Sort (actual time=0.003..0.003 rows=0 loops=51) + Sort Key: a."position" + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Recheck Cond: (message_id = m.id) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Index Cond: (message_id = m.id) + Buffers: shared hit=51 + Planning: + Buffers: shared hit=63 + Planning Time: 0.157 ms + Execution Time: 0.242 ms + + +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Limit (actual time=0.878..0.884 rows=51 loops=1) + Buffers: shared hit=258 + -> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1) + Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) + Buffers: shared hit=258 + Planning Time: 0.037 ms + Execution Time: 0.897 ms + + +=== 4. Query 2 without the index (index is dropped only inside this transaction) + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual time=1.097..1.101 rows=51 loops=1) + Buffers: shared hit=261 + -> Sort (actual time=1.097..1.098 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 31kB + Buffers: shared hit=261 + -> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1) + Filter: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) + Rows Removed by Filter: 15146 + Buffers: shared hit=261 + Planning: + Buffers: shared hit=4 + Planning Time: 0.051 ms + Execution Time: 1.115 ms + + +=== ROLLBACK: test data removed, index is back + explain_users_left +-------------------- + 0 + diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 0000000..597939b --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,43 @@ +# explain: EXPLAIN of the history page (full plans: explain-history.txt) +# date: 2026-10-21 21:09:14, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/explain.py +# exit code: 0, 1.1 s + +=== Data volume +=== History index +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) + Buffers: shared hit=5 + Buffers: shared hit=6 + Execution Time: 0.020 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + Buffers: shared hit=6 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Buffers: shared hit=6 + Execution Time: 0.031 ms +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + Buffers: shared hit=60 + -> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1) + Buffers: shared hit=60 + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + Buffers: shared hit=63 + Execution Time: 0.242 ms +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + Buffers: shared hit=258 + -> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1) + Buffers: shared hit=258 + Execution Time: 0.897 ms +=== 4. Query 2 without the index (index is dropped only inside this transaction) + Buffers: shared hit=261 + Buffers: shared hit=261 + -> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1) + Buffers: shared hit=261 + Buffers: shared hit=4 + Execution Time: 1.115 ms +=== ROLLBACK: test data removed, index is back +PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..9bb22e8 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,33 @@ +# failover: public failover test for every replica, file downloaded via the other one (make test-failover) +# date: 2026-10-21 21:09:54, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/run_failover.py +# exit code: 0, 39.7 s + +hook: scripts\stop-one.cmd; api replicas: ['11a93acecaca', '6128eb268c44'] +=== run 1: api containers before +SERVICE CONTAINER ID STATUS +api 11a93acecaca Up 36 seconds (healthy) +api 6128eb268c44 Up 36 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.39} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 11a93acecaca Exited (0) 11 seconds ago +api 6128eb268c44 Up 53 seconds (healthy) +the hook stopped 11a93acecaca (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 11a93acecaca -> healthy and back in HAProxy +=== run 2: api containers before +SERVICE CONTAINER ID STATUS +api 11a93acecaca Up 2 seconds (healthy) +api 6128eb268c44 Up 56 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 4.328} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 11a93acecaca Up 18 seconds (healthy) +api 6128eb268c44 Exited (0) 11 seconds ago +the hook stopped 6128eb268c44 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 6128eb268c44 -> healthy and back in HAProxy +PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 0000000..28d1d69 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,29 @@ +# failures: Redis and PostgreSQL outages via the balancer +# date: 2026-10-21 21:11:20, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_failures.py +# exit code: 0, 55.6 s + +stopping redis +/health/live -> 200, /health/ready -> 503 (8016 ms) +valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked +unknown token -> 503, not 200: nobody is trusted without a check +no token -> 401 +login -> 503: there is nowhere to store the session +registration -> 201: it needs only PostgreSQL +starting redis +after start: the old token works, confirmed messages are in place +PASS: Redis outage gives 503 and does not let anyone in without a session check +killing redis with SIGKILL (docker compose kill redis): no graceful shutdown +right after the crash: GET /users/me -> 503 +starting redis +after start: the session issued before the crash still works (AOF, appendfsync always) +PASS: sessions survive a Redis crash, not only a clean restart +stopping postgres +/health/live -> 200, /health/ready -> 503 (1016 ms) +valid token: users/me, chats, history, new message -> 503 unavailable, no details +no token -> 401, unknown token -> 401: anonymous users are not let in +login -> 503, registration -> 503 +starting postgres +after start: the old token works, confirmed messages are in place +PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out +3 passed, 0 failed diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..205a798 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-10-21 21:08:52, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_idempotency.py +# exit code: 0, 2.4 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'6128eb268c44': 10, '11a93acecaca': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..c4db3af --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 4, + "successful_requests": 7379, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 245.844, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 0000000..8529586 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-10-21 21:10:24, commit: 2dbed80 + uncommitted changes +# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.4 s + +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 4, + "successful_requests": 7379, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 245.844, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..f23f416 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,24 @@ +# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence) +# date: 2026-10-21 21:12:49, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/run_persistence.py +# exit code: 0, 40.2 s + +before: container, ID, started at + messenger-lab4-api-1 11a93acecaca 2026-10-21T18:11:37.067348364Z + messenger-lab4-api-2 6128eb268c44 2026-10-21T18:11:36.916459116Z + messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:11:33.313531682Z + messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:11:40.981519931Z + messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:11:22.482229396Z + messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:11:22.626999402Z + messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:11:32.264154477Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after (hook + waiting for readiness took 39.2 s): container, ID, started at, restarted? + messenger-lab4-api-1 11a93acecaca 2026-10-21T18:12:46.288933033Z same container: True, restarted: True + messenger-lab4-api-2 6128eb268c44 2026-10-21T18:12:46.174283308Z same container: True, restarted: True + messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:12:42.503993973Z same container: True, restarted: True + messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:12:47.684809415Z same container: True, restarted: True + messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:12:11.527270274Z same container: True, restarted: True + messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:12:11.368416107Z same container: True, restarted: True + messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:12:21.264135454Z same container: True, restarted: True +PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, new start time); data, the attachment bytes and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 0000000..7e5c4df --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-10-21 21:08:56, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_race.py +# exit code: 0, 1.2 s + +sent 100 messages from 10 clients in 0.22 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/redis.txt b/evidence/redis.txt new file mode 100644 index 0000000..b118ac4 --- /dev/null +++ b/evidence/redis.txt @@ -0,0 +1,14 @@ +# redis: Redis as the session store +# date: 2026-10-21 21:09:13, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/redis_info.py +# exit code: 0, 2.8 s + +session key: session:f4f1fe0235d139de...; value is the user id: True +TTL of the key, seconds: 3599 +sessions in Redis: 126 +config appendonly = yes +config appendfsync = always +config maxmemory-policy = noeviction +aof_enabled:1 +aof_last_write_status:ok +without password: NOAUTH Authentication required. diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt new file mode 100644 index 0000000..d118d61 --- /dev/null +++ b/evidence/s3-security.txt @@ -0,0 +1,44 @@ +# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap +# date: 2026-10-21 21:03:14, commit: 2dbed80 +# command: python.exe student-tests/test_s3_security.py +# exit code: 0, 9.5 s + +anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous head_object directly to S3 (from the data network) -> 403 +anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied +anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied +the same file through the API for a chat member -> 200 +PASS: the bucket is private: no anonymous read, list, write or policy read +API key: list buckets -> ['messenger-attachments'] +API key: get own object -> allowed +API key: put object -> allowed +API key: delete object -> allowed +API key: list bucket -> allowed +API key: create another bucket -> denied: AccessDenied +API key: make bucket public (put policy) -> denied: NotImplemented +API key: read bucket policy -> denied: NoSuchBucketPolicy +API key: delete bucket -> denied: AccessDenied +API key: change versioning -> denied: NotImplemented +PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied +compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand) +127.0.0.1:8333 from the host -> connection refused +127.0.0.1:9333 from the host -> connection refused +127.0.0.1:8888 from the host -> connection refused +127.0.0.1:19333 from the host -> connection refused +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init +s3-init: s3-init: bucket messenger-attachments exists +s3-init: s3-init: api key can put/get/list/delete objects of its bucket +s3-init: s3-init: api key cannot create buckets (AccessDenied) +s3-init: s3-init: bucket is private (anonymous list/get/put -> [403]) +after the repeated bootstrap: old file downloads, new upload works +PASS: bucket/user/policy bootstrap can be repeated safely +4 passed, 0 failed diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..95abed9 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test), with attachment +# date: 2026-10-21 21:03:01, commit: 2dbed80 +# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 0.6 s + +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 0000000..9b6415d --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,40 @@ +# stack: stand layout: replicas, ports, limits, networks, secrets, S3 +# date: 2026-10-21 21:03:00, commit: 2dbed80 +# command: python.exe student-tests/check_stack.py +# exit code: 0, 3.0 s + +host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 +Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop + +SERVICE CONTAINER ID STATUS PORTS +api 11a93acecaca Up 20 seconds (healthy) 8080/tcp +api 6128eb268c44 Up 20 seconds (healthy) 8080/tcp +janitor baefc0e97dff Up 20 seconds +lb 03697e5fe34c Up 18 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +migrate aca090330db4 Exited (0) 22 seconds ago +postgres f7a6a1ce25a7 Up 2 minutes (healthy) +redis b4523b39a0fe Up 2 minutes (healthy) +s3 e7cbe6ddfa4a Up 2 minutes (healthy) +s3-init 60719e674e18 Exited (0) 21 seconds ago + +container image cpus memory host ports networks +messenger-lab4-api-1 messenger-lab4-api 1.00 512 MiB - data,edge +messenger-lab4-api-2 messenger-lab4-api 1.00 512 MiB - data,edge +messenger-lab4-janitor-1 messenger-lab4-janitor 0.25 192 MiB - data +messenger-lab4-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge +messenger-lab4-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data +messenger-lab4-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data +messenger-lab4-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data + +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] + +through http://127.0.0.1:8080 one session is served by 2 replicas: ['11a93acecaca', '6128eb268c44'] +/health/ready -> 200 from 11a93acecaca +PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor only in the internal data network, S3 admin keys only in s3 and s3-init diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 0000000..9ac2c0f --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-10-21 21:08:55, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_api.py +# exit code: 0, 2.7 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/summary.md b/evidence/summary.md new file mode 100644 index 0000000..5df443a --- /dev/null +++ b/evidence/summary.md @@ -0,0 +1,40 @@ +# Сводка проверок + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 40 | evidence/persistence.txt | + +# Замеры + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 | +| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 | +| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 | +| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 | +| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..3050178 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,124 @@ +{ + "date": "2026-10-21 21:08", + "api_limit_mib": 512, + "api_limit_cpus": 1.0, + "runs": [ + { + "scenario": "10 x ~9.5 MiB PNG (noise)", + "concurrency": 10, + "file_bytes": [ + 9953047 + ], + "total_mib": 94.9, + "all_201": true, + "wall_seconds": 1.52, + "latency_s": { + "p50": 1.38, + "max": 1.52 + }, + "sampler_rounds": 9, + "replicas": [ + { + "replica": "11a93acecaca", + "uploads_served": 5, + "rss_before_mib": 93.5, + "rss_peak_mib": 106.2, + "rss_growth_mib": 12.6, + "rss_peak_sampled_mib": 106.2, + "cgroup_peak_mib": 134.1, + "cgroup_peak_sampled_mib": 127.7 + }, + { + "replica": "6128eb268c44", + "uploads_served": 5, + "rss_before_mib": 94.8, + "rss_peak_mib": 105.3, + "rss_growth_mib": 10.5, + "rss_peak_sampled_mib": 105.3, + "cgroup_peak_mib": 135.2, + "cgroup_peak_sampled_mib": 131.4 + } + ] + }, + { + "scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", + "concurrency": 10, + "file_bytes": [ + 116714 + ], + "total_mib": 1.1, + "all_201": true, + "wall_seconds": 1.09, + "latency_s": { + "p50": 0.85, + "max": 1.09 + }, + "sampler_rounds": 8, + "replicas": [ + { + "replica": "11a93acecaca", + "uploads_served": 5, + "rss_before_mib": 93.2, + "rss_peak_mib": 96.6, + "rss_growth_mib": 3.4, + "rss_peak_sampled_mib": 96.6, + "cgroup_peak_mib": 84.3, + "cgroup_peak_sampled_mib": 77.3 + }, + { + "replica": "6128eb268c44", + "uploads_served": 5, + "rss_before_mib": 93.1, + "rss_peak_mib": 94.7, + "rss_growth_mib": 1.6, + "rss_peak_sampled_mib": 94.6, + "cgroup_peak_mib": 86.8, + "cgroup_peak_sampled_mib": 77.3 + } + ] + }, + { + "scenario": "20 mixed: large + 40 Mpx + JPEG", + "concurrency": 20, + "file_bytes": [ + 1492, + 116714, + 9953047 + ], + "total_mib": 133.2, + "all_201": true, + "wall_seconds": 1.77, + "latency_s": { + "p50": 1.37, + "max": 1.77 + }, + "sampler_rounds": 10, + "replicas": [ + { + "replica": "11a93acecaca", + "uploads_served": 10, + "rss_before_mib": 94.8, + "rss_peak_mib": 121.7, + "rss_growth_mib": 26.9, + "rss_peak_sampled_mib": 121.7, + "cgroup_peak_mib": 181.9, + "cgroup_peak_sampled_mib": 180.6 + }, + { + "replica": "6128eb268c44", + "uploads_served": 10, + "rss_before_mib": 95.0, + "rss_peak_mib": 116.1, + "rss_growth_mib": 21.0, + "rss_peak_sampled_mib": 116.1, + "cgroup_peak_mib": 155.6, + "cgroup_peak_sampled_mib": 155.6 + } + ] + } + ], + "single_file_extra_rss_mib": { + "naive": 161, + "streaming": 0 + } +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 0000000..cdcb73f --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,13 @@ +# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas +# date: 2026-10-21 21:08:48, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json +# exit code: 0, 34.2 s + +client: Python 3.11.1 on Windows 10 +preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)... +large: 9953047 bytes; 40 Mpx: 116714 bytes +{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_201": true, "wall_seconds": 1.52, "latency_s": {"p50": 1.38, "max": 1.52}, "sampler_rounds": 9, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.5, "rss_peak_mib": 106.2, "rss_growth_mib": 12.6, "rss_peak_sampled_mib": 106.2, "cgroup_peak_mib": 134.1, "cgroup_peak_sampled_mib": 127.7}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 94.8, "rss_peak_mib": 105.3, "rss_growth_mib": 10.5, "rss_peak_sampled_mib": 105.3, "cgroup_peak_mib": 135.2, "cgroup_peak_sampled_mib": 131.4}]} +{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_201": true, "wall_seconds": 1.09, "latency_s": {"p50": 0.85, "max": 1.09}, "sampler_rounds": 8, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.2, "rss_peak_mib": 96.6, "rss_growth_mib": 3.4, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 84.3, "cgroup_peak_sampled_mib": 77.3}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 93.1, "rss_peak_mib": 94.7, "rss_growth_mib": 1.6, "rss_peak_sampled_mib": 94.6, "cgroup_peak_mib": 86.8, "cgroup_peak_sampled_mib": 77.3}]} +{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_201": true, "wall_seconds": 1.77, "latency_s": {"p50": 1.37, "max": 1.77}, "sampler_rounds": 10, "replicas": [{"replica": "11a93acecaca", "uploads_served": 10, "rss_before_mib": 94.8, "rss_peak_mib": 121.7, "rss_growth_mib": 26.9, "rss_peak_sampled_mib": 121.7, "cgroup_peak_mib": 181.9, "cgroup_peak_sampled_mib": 180.6}, {"replica": "6128eb268c44", "uploads_served": 10, "rss_before_mib": 95.0, "rss_peak_mib": 116.1, "rss_growth_mib": 21.0, "rss_peak_sampled_mib": 116.1, "cgroup_peak_mib": 155.6, "cgroup_peak_sampled_mib": 155.6}]} +one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB +PASS: all uploads succeeded; largest RSS growth of one replica: 26.9 MiB (limit 512 MiB); results in evidence/upload-memory.json diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..1605eed --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,77 @@ +# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики). +# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить +# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky). + +global + log stdout format raw local0 info + maxconn 4096 + # При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с. + hard-stop-after 10s + +# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик. +# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s # перечитывать список реплик раз в 2 с + hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlog-normal # в лог только ошибки, а не каждый запрос + timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос + timeout client 30s + # Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка + # при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт, + # сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT). + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода. + # Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key. + retries 2 + option redispatch 1 + +frontend api_in + bind :8080 + option forwardfor + default_backend api + +backend api + balance roundrobin + # Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis. + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT. + # Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP. + # observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки. + server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + # Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis + # или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате + # контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает. + # Пока готова хоть одна реплика, эти записи не используются. + # allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend). + server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + option allbackups + +# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1). +# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда. +frontend stats + bind :8404 + monitor-uri /lb-health + monitor fail if { nbsrv(api) lt 1 } + stats enable + stats uri /stats + stats refresh 2s diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 0000000..33c8c08 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,13 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +appendonly yes +appendfsync always +save "" + +maxmemory 64mb +maxmemory-policy noeviction + +include /run/secrets/redis_auth diff --git a/infra/s3-init.sh b/infra/s3-init.sh new file mode 100644 index 0000000..ab0adae --- /dev/null +++ b/infra/s3-init.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# Bootstrap of the S3 storage (MinIO), run by the one-shot service s3-init before api starts. +# Idempotent: every step can be repeated (docker compose run --rm s3-init) without losing data. +# 1. private bucket S3_BUCKET (no anonymous access, versioning off); +# 2. policy messenger-api: only Get/Put/Delete/List objects of this bucket (JSON below); +# 3. separate user for api/janitor with this policy. Root credentials stay in s3 and s3-init only. +# Secrets are read from /run/secrets and are never printed. +set -eu + +BUCKET="${S3_BUCKET:?S3_BUCKET is not set}" +ROOT_USER="$(cat /run/secrets/s3_root_user)" +ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)" +API_KEY="$(cat /run/secrets/s3_access_key_id)" +API_SECRET="$(cat /run/secrets/s3_secret_access_key)" +POLICY=messenger-api +export MC_CONFIG_DIR=/tmp/mc + +# Least privilege for the api key: objects of ONE bucket. No bucket admin, policies, users or other buckets. +cat > /tmp/policy.json </dev/null 2>&1; do + i=$((i + 1)) + [ "$i" -ge 30 ] && { echo "s3-init: MinIO is not reachable"; exit 1; } + sleep 1 +done + +mc mb --ignore-existing "local/${BUCKET}" >/dev/null +mc anonymous set none "local/${BUCKET}" >/dev/null +echo "s3-init: bucket ${BUCKET} exists" + +mc admin policy create local "$POLICY" /tmp/policy.json >/dev/null +echo "s3-init: policy ${POLICY} created/updated" + +mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null +info="$(mc admin user info local "$API_KEY" 2>/dev/null || true)" +case "$info" in + *"$POLICY"*) ;; + *) mc admin policy attach local "$POLICY" --user "$API_KEY" >/dev/null ;; +esac +echo "s3-init: api user exists with policy ${POLICY} only" + +access="$(mc anonymous get "local/${BUCKET}")" +case "$access" in + *private*) echo "s3-init: bucket is private (anonymous access: none)" ;; + *) echo "s3-init: bucket is NOT private: $access"; exit 1 ;; +esac diff --git a/janitor.py b/janitor.py new file mode 100644 index 0000000..58bee8f --- /dev/null +++ b/janitor.py @@ -0,0 +1,162 @@ +"""Очистка незавершённых загрузок и сверка PostgreSQL <-> S3. + +Запускается отдельным сервисом `janitor` (раз в JANITOR_INTERVAL_SECONDS) или вручную: + + python janitor.py --once [--dry-run] [--grace-seconds N] + +Проход состоит из двух шагов. Оба трогают только то, что старше grace period +(ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч, минимум 2 x UPLOAD_TIMEOUT_SECONDS): загрузка, +которая ещё идёт, столько не живёт, поэтому свежие и готовые вложения не удаляются. + +1. Брошенные намерения: строки state=uploading старше grace. Для каждой (под FOR UPDATE SKIP LOCKED) + удалить объект в S3 (идемпотентно) и затем строку. Если S3 недоступен, строка остаётся + до следующего прохода — указатель на объект не теряется. +2. Сирота в S3: объект под originals/ старше grace, для которого в БД нет строки ни в каком + состоянии. API кладёт объект только после COMMIT строки-намерения, поэтому у живой загрузки + строка всегда есть. Удаляется. + Заодно отчёт: ready-строки, чьего объекта нет в bucket (missing). Их не трогаем — это потеря + данных, её разбирает человек (восстановление из резервной копии), а не автоматическая очистка. + +Одновременно работает только один janitor: pg_try_advisory_lock. +""" +import argparse +import json +import logging +import signal +import threading +import time +from datetime import datetime, timedelta, timezone + +import psycopg +from psycopg.rows import dict_row + +import settings +import storage + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.janitor") + +JANITOR_LOCK = 20260404 +BATCH = 100 +STOP = threading.Event() + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]: + """Шаг 1. Возвращает (найдено, удалено).""" + found = deleted = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, object_key FROM attachments + WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + FOR UPDATE SKIP LOCKED""", + (grace, BATCH), + ).fetchall() + found += len(rows) + if dry_run or not rows: + return found, deleted + for row in rows: + storage.delete(row["object_key"]) # недоступен S3 -> исключение, транзакция откатится + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],)) + deleted += 1 + log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"]) + if len(rows) < BATCH: + return found, deleted + return found, deleted + + +def sweep_objects(conn, grace: int, dry_run: bool) -> dict: + """Шаг 2. Обход bucket постранично; сверка ключей с БД пачками по 1000.""" + started = datetime.now(timezone.utc) + cutoff = started - timedelta(seconds=grace) + scanned = orphans = orphans_deleted = 0 + listed = set() + page: list[tuple[str, datetime]] = [] + + def check(page_items): + nonlocal orphans, orphans_deleted + keys = [key for key, _ in page_items] + known = {r["object_key"] for r in conn.execute( + "SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,)).fetchall()} + for key, modified in page_items: + if key in known or modified >= cutoff: + continue + orphans += 1 + if not dry_run: + # Повторная проверка прямо перед удалением: строка могла появиться между запросами. + if conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (key,)).fetchone() is None: + storage.delete(key) + orphans_deleted += 1 + log.info("deleted orphan object %s (no row in PostgreSQL)", key) + + for key, modified, _size in storage.list_objects(): + scanned += 1 + listed.add(key) + page.append((key, modified)) + if len(page) >= 1000: + check(page) + page = [] + if page: + check(page) + + missing = [str(r["id"]) for r in conn.execute( + "SELECT id, object_key FROM attachments WHERE state = 'ready' AND updated_at < %s", (started,)).fetchall() + if r["object_key"] not in listed] + if missing: + log.error("ready attachments without object in S3 (not deleted, needs restore): %s", ", ".join(missing[:20])) + return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted, + "missing_objects": len(missing)} + + +def run_once(grace: int, dry_run: bool) -> dict | None: + started = time.monotonic() + with connect() as conn: + if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]: + log.info("another janitor is running, skip") + return None + try: + found, deleted = reclaim_stale_intents(conn, grace, dry_run) + report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found, + "stale_uploads_deleted": deleted, **sweep_objects(conn, grace, dry_run)} + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,)) + report["seconds"] = round(time.monotonic() - started, 3) + return report + + +def main(): + parser = argparse.ArgumentParser(description="Reclaim abandoned uploads and orphan S3 objects") + parser.add_argument("--once", action="store_true", help="one pass and exit") + parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing") + parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS) + args = parser.parse_args() + if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS: + parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} " + "(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted") + + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + while True: + try: + report = run_once(args.grace_seconds, args.dry_run) + if report is not None: + print(json.dumps(report), flush=True) + failed = False + except (psycopg.Error, storage.StorageUnavailable) as exc: + log.warning("pass failed, will retry: %s", type(exc).__name__) + failed = True + if args.once: + raise SystemExit(1 if failed else 0) + if STOP.wait(settings.JANITOR_INTERVAL_SECONDS): + return + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py new file mode 100644 index 0000000..0cfe9b9 --- /dev/null +++ b/main.py @@ -0,0 +1,583 @@ +import base64 +import binascii +import hashlib +import json +import logging +import re +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path +from typing import Literal +from urllib.parse import quote +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +import attachments +import auth +import db +import settings +import storage +import uploads +from errors import ApiError, bad_request, unauthorized, unavailable + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger") + +DRAINING = threading.Event() + + +class Server(uvicorn.Server): + """Остановка по SIGTERM в два шага. + + 1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS + HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы. + 2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы + (не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM, + получал бы 502 от балансировщика. + Повторный сигнал останавливает сразу. + """ + + def handle_exit(self, sig, frame): + if DRAINING.is_set(): + return super().handle_exit(sig, frame) + DRAINING.set() + log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s", + sig, settings.SHUTDOWN_DRAIN_SECONDS) + threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings.check() + # Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему. + await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS) + # Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3. + Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True) + db.pool.open() + log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s", + settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, settings.SESSION_TTL_SECONDS, + settings.S3_BUCKET) + yield + db.pool.close() + auth.close() + log.info("Stopped: connections closed, data stays in PostgreSQL and Redis") + + +app = FastAPI(lifespan=lifespan) + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH) + + @field_validator("display_name", "password") + @classmethod + def validate_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("value cannot be blank") + return value + + +class User(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(BaseModel): + token: str + token_type: str + expires_at: datetime + user: User + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title cannot be empty") + return value + + +class Chat(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class ChatPage(BaseModel): + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + # С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("attachment_ids must be unique") + if not self.attachment_ids and not self.text.strip(): + raise ValueError("text cannot be empty without attachments") + return self + + +class Message(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: list[UUID] + + +class Variant(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["ready"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + +class MessagePage(BaseModel): + items: list[Message] + next_cursor: str | None + + +def error_response( + request: Request, + code: str, + message: str, + status_code: int, + headers: dict | None = None, +): + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers={"X-Request-Id": request_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + return error_response( + request, + "invalid_request", + "Invalid request data", + 400, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Not found", 404) + return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def general_exception_handler( + request: Request, + exc: Exception, +): + return error_response( + request, + "internal_error", + "Internal server error", + 500, + ) + +PUBLIC_ENDPOINTS = { + ("POST", "/api/v1/users"), + ("POST", "/api/v1/auth/sessions"), +} + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + path = request.url.path + + try: + if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS: + request.state.session = await run_in_threadpool( + auth.check_bearer, request.headers.get("Authorization") + ) + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + except Exception: + log.exception("Unhandled error, request_id=%s", request_id) + response = error_response(request, "internal_error", "Internal server error", 500) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила + # Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки. + level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO + log.log( + level, + "%s %s -> %s (%.1f ms) request_id=%s", + request.method, path, response.status_code, + (time.perf_counter() - started) * 1000, request_id, + ) + return response + + +def current_user_id(request: Request) -> UUID: + session = getattr(request.state, "session", None) + if session is None: + raise unauthorized("Bearer token is required") + return session.user_id + + +def encode_cursor( + created_at: datetime, + object_id: UUID, + *, + chat_id: UUID | None = None, + user_id: UUID | None = None, +) -> str: + data = { + "created_at": created_at.isoformat(), + "id": str(object_id), + } + + if chat_id is not None: + data["chat_id"] = str(chat_id) + + if user_id is not None: + data["user_id"] = str(user_id) + + raw = json.dumps(data).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode_cursor(cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + data = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(data["created_at"]) + + if created_at.tzinfo is None: + raise ValueError("Invalid cursor") + + if not isinstance(data["id"], str): + raise ValueError("Invalid cursor") + + object_id = UUID(data["id"]) + + chat_id = UUID(data["chat_id"]) if "chat_id" in data else None + user_id = UUID(data["user_id"]) if "user_id" in data else None + + return created_at, object_id, chat_id, user_id + + except ( + ValueError, + KeyError, + TypeError, + AttributeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("Invalid cursor") + + +def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): + if cursor is None: + return None + try: + created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) + except ValueError: + raise bad_request("Invalid cursor") from None + if cursor_chat_id != chat_id or cursor_user_id != user_id: + raise bad_request("Invalid cursor") + return created_at, object_id + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + if DRAINING.is_set(): + raise unavailable("Instance is shutting down") + db.ping() + auth.ping_redis() + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=User, status_code=201) +def create_user(user_data: CreateUser): + password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется + return db.create_user(user_data.username, user_data.display_name, password_hash) + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +def create_session( + response: Response, + authorization: str | None = Header(default=None), +): + credentials = auth.parse_basic(authorization) + if credentials is None: + raise auth.login_failed() + username, password = credentials + + found = None + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: + found = db.get_user_with_password(username) + if not auth.password_matches(found["password_hash"] if found else None, password): + raise auth.login_failed() + + token, expires_at = auth.create_session(found["id"]) + response.headers["Cache-Control"] = "no-store" + user = {key: found[key] for key in ("id", "username", "display_name", "created_at")} + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + current_user_id(request) + auth.delete_session(request.state.session) + return Response(status_code=204) + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(request: Request): + user = db.get_user(current_user_id(request)) + if user is None: + raise unauthorized("Session is invalid or expired") + return user + + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, chat_data: CreateChat): + user_id = current_user_id(request) + + if user_id not in chat_data.member_ids: + raise bad_request("Creator must be a chat member") + + if len(chat_data.member_ids) != len(set(chat_data.member_ids)): + raise bad_request("Duplicate member IDs") + return db.create_chat(chat_data.title, chat_data.member_ids) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, user_id=user_id) + + rows = db.list_chats(user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id) + + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID): + return db.get_chat(chat_id, current_user_id(request)) + + +def request_fingerprint(data: BaseModel) -> bytes: + """SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют. + + Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []} + равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними. + """ + body = data.model_dump(mode="json") + if not body.get("attachment_ids"): + body.pop("attachment_ids", None) + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + response_model=Message, + status_code=201, +) +def create_message( + request: Request, + message_data: CreateMessage, + chat_id: UUID, + idempotency_key: str | None = Header(default=None), +): + # С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику) + # вернёт то же сообщение, а не создаст второе. См. db.add_message. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -") + return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids, + idempotency_key, request_fingerprint(message_data)) + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +def list_messages( + request: Request, + chat_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, chat_id=chat_id) + + rows = db.list_messages(chat_id, user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id) + + return MessagePage(items=items, next_cursor=next_cursor) + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=Attachment, + status_code=201, +) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: str | None = Query(default=None), +): + """Синхронная загрузка PNG/JPEG сырыми байтами. 201 — только когда байты в S3 и строка ready в БД. + + Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415) + -> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413), + проверка содержимого (415/413) и запись (attachments.store). + """ + user_id = current_user_id(request) + try: + await run_in_threadpool(db.check_member, chat_id, user_id) + name = uploads.validate_filename(filename) + declared = uploads.declared_type(request.headers.get("content-type")) + uploads.declared_length(request.headers.get("content-length")) + except ApiError: + await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке + raise + upload = await uploads.receive(request) + try: + image = await run_in_threadpool(uploads.inspect, upload, declared) + row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image) + finally: + upload.file.close() # временный файл удаляется сразу; оригинал остаётся только в S3 + return attachments.public(row) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public(db.get_attachment(attachment_id, current_user_id(request))) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + request: Request, + attachment_id: UUID, + variant: Literal["original"] = Query(default="original"), +): + """Байты оригинала через API (без redirect): сначала проверка участника чата, потом S3.""" + row = db.get_attachment(attachment_id, current_user_id(request)) + try: + stream = storage.open_read(row["object_key"]) + except storage.ObjectMissing: + log.error("Object of ready attachment %s is missing in S3", row["id"]) + raise unavailable("File is temporarily unavailable") from None + except storage.StorageUnavailable: + raise unavailable("File storage is temporarily unavailable") from None + headers = { + "Content-Length": str(row["size_bytes"]), + "Content-Disposition": "inline; filename*=UTF-8''" + quote(row["filename"], safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + row["sha256"] + '"', + } + return StreamingResponse(storage.iter_chunks(stream), media_type=row["content_type"], headers=headers) + + +if __name__ == "__main__": + Server(uvicorn.Config( + app, + host=settings.APP_HOST, + port=settings.APP_PORT, + timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS, + access_log=False, + )).run() diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..e95f414 --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +import logging + +import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + +if __name__ == "__main__": + db.wait_and_migrate() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..8b981d0 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,33 @@ + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 0000000..73f0c84 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..5770e1d --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,11 @@ +ALTER TABLE messages ADD COLUMN idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); + +ALTER TABLE messages ADD COLUMN request_hash bytea + CHECK (octet_length(request_hash) = 32); + +ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together + CHECK ((idempotency_key IS NULL) = (request_hash IS NULL)); + +ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key + UNIQUE (chat_id, sender_id, idempotency_key); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..35697dd --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,48 @@ +-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением. +-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать. + +-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата. +ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + -- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит. + object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'), + -- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна; + -- ready: объект в S3 сохранён, можно читать и прикреплять. + state text NOT NULL CHECK (state IN ('uploading', 'ready')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (width::bigint * height <= 40000000), + UNIQUE (id, chat_id), + -- владелец на момент загрузки был участником именно этого чата + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); + +-- janitor ищет старые незавершённые загрузки +CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + -- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id); + +-- С lab4 текст может быть пустым, если есть вложение (проверяет API). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..eb8970d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +fastapi==0.141.1 +uvicorn==0.54.0 +pydantic==2.13.5 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==6.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +pycparser==3.1 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +boto3==1.43.116 +botocore==1.43.116 +s3transfer==0.19.3 +jmespath==1.1.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +urllib3==2.8.0 +pillow==12.3.0 diff --git a/s3_init.py b/s3_init.py new file mode 100644 index 0000000..99489bc --- /dev/null +++ b/s3_init.py @@ -0,0 +1,103 @@ +"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init. + +Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает. +Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py): + admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init; + messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor. +Значения ключей не печатаются. +""" +import logging +import sys +import time +import urllib.error +import urllib.request +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +logging.basicConfig(level=logging.WARNING) + + +def client(key_id: str, secret: str): + return boto3.client( + "s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION, + aws_access_key_id=key_id, aws_secret_access_key=secret, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2, + read_timeout=10, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", response_checksum_validation="when_required"), + ) + + +def code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "?") + + +def anonymous_status(url: str, method: str = "GET") -> int: + try: + with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply: + return reply.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + bucket = settings.S3_BUCKET + admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"), + settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY) + + deadline = time.monotonic() + 90 + while True: + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if time.monotonic() > deadline: + print(f"s3-init: S3 is not ready: {type(exc).__name__}") + return 1 + time.sleep(1) + + try: + admin.create_bucket(Bucket=bucket) + print(f"s3-init: bucket {bucket} created") + except ClientError as exc: + if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + print(f"s3-init: cannot create bucket: {code(exc)}") + return 1 + print(f"s3-init: bucket {bucket} exists") + + # Ключ API работает со своим bucket ... + probe = "originals/s3-init-probe-" + uuid.uuid4().hex + api.put_object(Bucket=bucket, Key=probe, Body=b"probe") + api.get_object(Bucket=bucket, Key=probe)["Body"].read() + api.list_objects_v2(Bucket=bucket, MaxKeys=1) + api.delete_object(Bucket=bucket, Key=probe) + print("s3-init: api key can put/get/list/delete objects of its bucket") + # ... и не может администрировать хранилище. + intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8] + try: + api.create_bucket(Bucket=intruder) + admin.delete_bucket(Bucket=intruder) + print("s3-init: api key must NOT be able to create buckets") + return 1 + except ClientError as exc: + print(f"s3-init: api key cannot create buckets ({code(exc)})") + + # Приватность: анонимные запросы без подписи. + base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket + statuses = {"list": anonymous_status(base + "?list-type=2"), + "get": anonymous_status(base + "/originals/any"), + "put": anonymous_status(base + "/originals/any", "PUT")} + if any(status not in (401, 403) for status in statuses.values()): + print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}") + return 1 + print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..b69f27f --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,77 @@ +import json +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + with open(SECRETS / name, "w", encoding="utf-8", newline="") as file: + file.write(value) + (SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore + + +def bucket_name() -> str: + """S3_BUCKET из .env (если задан), иначе значение по умолчанию.""" + env = ROOT / ".env" + if env.exists(): + for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("S3_BUCKET="): + return line.split("=", 1)[1].strip() + return "messenger-attachments" + + +def s3_config(bucket: str) -> str: + """Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket. + + Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects) + с суффиксом : действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет. + Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются. + """ + def read(name): + return (SECRETS / name).read_text(encoding="utf-8").strip() + config = {"identities": [ + {"name": "admin", + "credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ]} + return json.dumps(config, indent=2) + + +def main() -> None: + """Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes).""" + SECRETS.mkdir(exist_ok=True) + created = [] + if not (SECRETS / "postgres_password").exists(): + db_password = secrets.token_hex(24) + redis_password = secrets.token_hex(24) + write("postgres_password", db_password) + write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger") + write("cache_url", f"redis://:{redis_password}@redis:6379/0") + write("redis_auth", f"requirepass {redis_password}\n") + created += ["postgres_password", "database_url", "cache_url", "redis_auth"] + if not (SECRETS / "s3_root_password").exists(): + # admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init + write("s3_root_user", "root-" + secrets.token_hex(6)) + write("s3_root_password", secrets.token_hex(24)) + created += ["s3_root_user", "s3_root_password"] + if not (SECRETS / "s3_secret_access_key").exists(): + # отдельный ключ API с политикой только на свой bucket (создаёт s3-init) + write("s3_access_key_id", "api-" + secrets.token_hex(8)) + write("s3_secret_access_key", secrets.token_hex(24)) + created += ["s3_access_key_id", "s3_secret_access_key"] + if not (SECRETS / "s3_config.json").exists(): + write("s3_config.json", s3_config(bucket_name())) + created.append("s3_config.json") + if created: + print("Created in secrets/: " + ", ".join(created) + " (values are not printed).") + else: + print("secrets/ is complete, nothing changed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..29bb4e7 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,13 @@ +@echo off +rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then api and janitor, then the balancer. Volumes are kept. +rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +cd /d "%~dp0.." +docker compose restart postgres redis s3 || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 || exit /b 1 +docker compose restart api janitor || exit /b 1 +docker compose restart lb || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps api lb || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..d09ac37 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Restart hook for tests/resilience.py --mode persistence (Linux/macOS). +# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then all api replicas and janitor, then the balancer. +# Volumes are kept (no down -v), nothing is re-seeded. +# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres redis s3 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 +docker compose restart api janitor +docker compose restart lb +docker compose up -d --wait --no-recreate --no-build --no-deps api lb diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..8ffd0fa --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py. +python "%~dp0stop_one.py" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..cf4d724 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID. +# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd. +set -eu +exec python3 "$(dirname "$0")/stop_one.py" diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 0000000..5cdcce7 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STOP_TIMEOUT_SECONDS = 8 + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{12,64}", target): + print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr) + return 2 + running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + matches = [container for container in running if container.startswith(target)] + if len(matches) != 1: + print(f"no running api container with id {target}", file=sys.stderr) + return 1 + subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + print(f"stopped api container {matches[0][:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..0693054 --- /dev/null +++ b/settings.py @@ -0,0 +1,86 @@ +import os +import socket +from pathlib import Path + + +def env_str(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def env_int(name: str, default: int) -> int: + value = int(env_str(name, str(default))) + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value + + +def read_secret(file_variable: str, plain_variable: str) -> str: + path = env_str(file_variable, "") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return env_str(plain_variable, "") + + +APP_HOST = env_str("APP_HOST", "0.0.0.0") +APP_PORT = env_int("APP_PORT", 8080) +APP_ENV = env_str("APP_ENV", "lab") +APP_VERSION = env_str("APP_VERSION", "dev") + +INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname() + +SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600) +SESSION_STORE = env_str("SESSION_STORE", "redis") + +DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL") +DB_POOL_MIN = env_int("DB_POOL_MIN", 1) +DB_POOL_MAX = env_int("DB_POOL_MAX", 10) +DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3) +DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3) +DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + +REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL") +REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20) +REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500) + +MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations"))) +SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60) + +# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket. +S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333") +S3_REGION = env_str("S3_REGION", "us-east-1") +S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1) +S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5) +S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2) +S3_POOL_MAX = env_int("S3_POOL_MAX", 20) + +# Вложения: лимиты из контракта и защита памяти/времени. +MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10) +DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2) +UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads") + +# Очистка незавершённых загрузок (janitor.py). +ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600) +JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300) + +SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3) +SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10) + + +def check() -> None: + """Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе.""" + if not DATABASE_URL: + raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE") + if not REDIS_URL: + raise RuntimeError("Redis address is not set: CACHE_URL_FILE") + if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY: + raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE") + if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS: + raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS") + if SESSION_STORE != "redis": + raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution") diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..1c88e16 --- /dev/null +++ b/storage.py @@ -0,0 +1,153 @@ +"""Приватное S3-хранилище (SeaweedFS) для байтов вложений. + +API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket +(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер, +имя файла пользователя в ключ не попадает. +""" +import base64 +import logging +from datetime import datetime +from typing import BinaryIO, Iterator + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +log = logging.getLogger("messenger.storage") + +ORIGINALS_PREFIX = "originals/" +READ_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей.""" + + +class ObjectMissing(Exception): + """Объекта нет в bucket (NoSuchKey / 404).""" + + +def _make_client(connect_timeout: float, read_timeout: float, attempts: int): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT, + region_name=settings.S3_REGION, + aws_access_key_id=settings.S3_ACCESS_KEY_ID or None, + aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, # http://s3:8333// + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую + max_pool_connections=settings.S3_POOL_MAX, + # Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно. + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + +_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS) +# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor. +_quick_client = _make_client(1, 2, 1) + + +def _is_missing(exc: ClientError) -> bool: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("NoSuchKey", "404", "NotFound") or status == 404 + + +def _unavailable(action: str, exc: Exception) -> StorageUnavailable: + detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else "" + log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail) + return StorageUnavailable() + + +def new_key(object_id: str) -> str: + return ORIGINALS_PREFIX + object_id + + +def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + """Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект.""" + body.seek(0) + try: + _client.put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("put", exc) from None + + +def open_read(key: str): + """GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после.""" + try: + return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"] + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing(key) from None + raise _unavailable("get", exc) from None + except BotoCoreError as exc: + raise _unavailable("get", exc) from None + + +def iter_chunks(stream) -> Iterator[bytes]: + """Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание.""" + try: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return + yield chunk + except (BotoCoreError, OSError) as exc: + # Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length). + log.warning("S3 stream interrupted: %s", type(exc).__name__) + raise + finally: + stream.close() + + +def delete(key: str, quick: bool = False) -> None: + """Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа.""" + try: + (_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("delete", exc) from None + + +def exists(key: str) -> bool: + try: + _client.head_object(Bucket=settings.S3_BUCKET, Key=key) + return True + except ClientError as exc: + if _is_missing(exc): + return False + raise _unavailable("head", exc) from None + except BotoCoreError as exc: + raise _unavailable("head", exc) from None + + +def list_objects(prefix: str = ORIGINALS_PREFIX) -> Iterator[tuple[str, datetime, int]]: + """Все объекты под prefix: (key, last_modified, size). Постранично, по 1000.""" + paginator = _client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"], item["Size"] + except (BotoCoreError, ClientError) as exc: + raise _unavailable("list", exc) from None + + +def ping() -> None: + try: + _client.head_bucket(Bucket=settings.S3_BUCKET) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("head_bucket", exc) from None diff --git a/student-tests/check_stack.py b/student-tests/check_stack.py new file mode 100644 index 0000000..3c91f0d --- /dev/null +++ b/student-tests/check_stack.py @@ -0,0 +1,78 @@ + +import json +import platform +import sys + +from common import call, check, compose_output, docker, new_user, seen_instances + + +def main(): + info = json.loads(docker("info", "--format", "{{json .}}")) + print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"client Python {platform.python_version()}") + print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, " + f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, " + f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}") + + print() + print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip()) + for one_shot in ("migrate", "s3-init"): + status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip() + check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0") + + config = json.loads(compose_output("config", "--format", "json")) + services = config["services"] + containers = compose_output("ps", "-q").split() + rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()] + print() + print("container image cpus memory host ports networks") + api_count = 0 + for row in sorted(rows, key=lambda r: r["Name"]): + labels = row["Config"]["Labels"] + service = labels.get("com.docker.compose.service") + limits = row["HostConfig"] + cpus = limits.get("NanoCpus", 0) / 1e9 + memory = limits.get("Memory", 0) // 2**20 + ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values() + for b in (bindings or [])}) + networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"]) + print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB " + f"{','.join(ports) or '-':<23} {','.join(networks)}") + check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit") + if service == "api": + api_count += 1 + check(not ports, "api must not publish ports: everything goes through the balancer") + if service == "lb": + check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only") + if service in ("postgres", "redis", "s3", "janitor"): + check(not ports and networks == ["data"], f"{service} must be only in the internal data network") + check(api_count >= 2, "need at least 2 api replicas") + check(all("container_name" not in s for s in services.values()), "container_name prevents scaling") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis") + + print() + for name, service in sorted(services.items()): + secrets = sorted(s["source"] for s in service.get("secrets", [])) + print(f"secrets of {name}: {secrets or '-'}") + check(not services["lb"].get("secrets"), "lb gets no secrets") + check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url") + for name in ("api", "janitor"): + names = {s["source"] for s in services[name]["secrets"]} + check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys") + check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key") + + user = new_user("stack") + served = seen_instances(user["token"]) + print() + print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}") + check(len(served) >= 2, "requests are not balanced") + health = call("GET", "/health/ready", 200) + print(f"/health/ready -> {health.status} from {health.instance}") + print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every " + f"container, postgres/redis/s3/janitor only in the internal data network, " + f"S3 admin keys only in s3 and s3-init") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 0000000..60dbdac --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,293 @@ + +import json +import os +import platform +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PY = sys.executable + +STEPS = [ + ("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]), + ("stack", "stand layout: replicas, ports, limits, networks, secrets, S3", [PY, "student-tests/check_stack.py"]), + ("smoke", "public smoke via the balancer (make test), with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]), + ("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids", + [PY, "student-tests/test_attachments.py"]), + ("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap", + [PY, "student-tests/test_s3_security.py"]), + ("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage", + [PY, "student-tests/test_consistency.py"]), + ("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas", + [PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]), + ("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]), + ("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]), + ("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]), + ("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]), + ("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]), + ("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]), + ("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]), + ("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)", + [PY, "student-tests/run_failover.py"]), + ("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL, + "--duration", "30", "--concurrency", "4", + "--output", "evidence/load.json"]), + ("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]), + ("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)", + [PY, "student-tests/run_persistence.py"]), +] + + +def git(*args): + try: + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "" + + +def commit(): + head = git("rev-parse", "--short", "HEAD") or "unknown" + changed = [line for line in git("status", "--porcelain").splitlines() + if not line[3:].startswith(("evidence/", "REPORT.md"))] + return head + (" + uncommitted changes" if changed else "") + + +def run_step(name, description, command): + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL) + print(f"=== {name}: {description}", flush=True) + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + seconds = time.monotonic() - started + output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n") + shown = " ".join(Path(part).name if part == PY else part for part in command) + header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n" + f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n") + (EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8") + tail = [line for line in output.splitlines() if line.strip()][-3:] + for line in tail: + print(" " + line) + print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True) + return result.returncode == 0, seconds + + +def load_json(name): + path = EVIDENCE / name + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + +def measurements(): + """Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md).""" + lines = [] + try: + info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True, + text=True, check=True).stdout) + docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM" + except (OSError, ValueError, subprocess.CalledProcessError): + docker_vm = "unknown" + lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона " + f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB " + f"каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, " + f"janitor 0,25 CPU / 192 MiB.") + lines.append("") + + failover = EVIDENCE / "failover.txt" + results = [] + if failover.exists(): + text = failover.read_text(encoding="utf-8") + stopped = re.findall(r"the hook stopped (\w+)", text) + results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)] + if results: + lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):") + lines.append("") + lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток " + "(повторены с тем же ключом) | Первая запись после hook, с |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for number, (result, container) in enumerate(zip(results, stopped), start=1): + lines.append(f"| {number} | {container} | {result['observed_instances_before']} | " + f"{result['successful_writes']} | {result['failed_attempts']} | " + f"{result['first_recovery_seconds']} |") + lines.append("") + + memory = load_json("upload-memory.json") + if memory: + lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики " + f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). " + f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:") + lines.append("") + lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | " + "Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for run in memory["runs"]: + for n, r in enumerate(run["replicas"]): + head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | " + f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |" + cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-" + lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | " + f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |") + naive = memory["single_file_extra_rss_mib"] + lines.append("") + lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в " + f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка " + f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в " + f"{memory['api_limit_mib']} MiB.") + lines.append("") + + metrics = load_json("failover-metrics.json") + if metrics: + lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи " + "без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):") + lines.append("") + lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок " + "(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | " + "Вернулась в балансировку после docker start, с |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for r in metrics["runs"]: + how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)" + codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-" + lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | " + f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | " + f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |") + lines.append("") + lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика " + "после остановки не ответила ни разу.") + lines.append("") + + rolling = load_json("rolling-update.json") + if rolling: + during = rolling["during"] + codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-" + lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> " + f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за " + f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, " + f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); " + f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено " + f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.") + lines.append("") + + load = load_json("load.json") + if load: + latency = load["successful_latency_ms"] + lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, " + f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных " + f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных " + f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.") + lines.append("") + + persistence = EVIDENCE / "persistence.txt" + restart_seconds = None + if persistence.exists(): + found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8")) + restart_seconds = found.group(1) if found else None + + impacts = [] + if metrics: + for mode, what, expected in ( + ("stop", "`docker stop` реплики (SIGTERM) под нагрузкой", + "ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"), + ("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой", + "единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")): + runs = [r for r in metrics["runs"] if r["mode"] == mode] + if runs: + impacts.append( + f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); " + f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | " + f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; " + f"потерь 0, дублей 0; реплика вернулась через " + f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |") + if failover.exists() and results: + impacts.append( + f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, " + f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: " + f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через " + f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |") + if rolling: + impacts.append( + f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | " + f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |") + if restart_seconds: + impacts.append( + f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и " + f"прежний токен живы | " + f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |") + consistency = EVIDENCE / "consistency.txt" + if consistency.exists(): + text = consistency.read_text(encoding="utf-8") + timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text) + if len(timings) >= 2: + impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, " + f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | " + f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |") + impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за " + f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |") + if "PASS: S3 written but DB not committed" in text: + impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | " + "вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | " + "GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | " + "мусора не осталось; повторный проход ничего не делает |") + if "PASS: an interrupted upload leaves no row" in text: + impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | " + "строк и объектов не прибавилось, временных файлов 0 | мгновенно |") + if impacts: + lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |") + lines.append("| --- | --- | --- | --- |") + lines.extend(impacts) + lines.append("") + return "\n".join(lines).strip() + + +def update_report(block): + report = ROOT / "REPORT.md" + text = report.read_text(encoding="utf-8") + begin, end = "", "" + if begin not in text or end not in text: + print("REPORT.md has no measurement markers, only evidence/summary.md was written") + return + head, rest = text.split(begin, 1) + _, tail = rest.split(end, 1) + report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8") + print("REPORT.md: measurement tables updated") + + +def main(): + args = sys.argv[1:] + names = [name for name, _, _ in STEPS] + if args[:1] == ["--from"]: + if len(args) < 2 or args[1] not in names: + raise SystemExit("usage: --from ; steps: " + ", ".join(names)) + selected = names[names.index(args[1]):] + else: + unknown = [a for a in args if a not in names] + if unknown: + raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names)) + selected = args or names + EVIDENCE.mkdir(exist_ok=True) + results = [] + for name, description, command in STEPS: + if name not in selected: + continue + ok, seconds = run_step(name, description, command) + results.append((name, description, ok, seconds)) + if not ok: + print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.") + print(f"After fixing: python student-tests/collect_evidence.py --from {name}") + break + block = measurements() + summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |", + "| --- | --- | --- | --- | --- |"] + summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results] + (EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8") + print("evidence/summary.md written") + passed = all(ok for _, _, ok, _ in results) + if passed: + update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 0000000..7fed6ff --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,324 @@ + +import base64 +import csv +import hashlib +import io +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +PASSWORD = "Test-password-123" + + +class Response: + def __init__(self, status, headers, data): + self.status = status + self.headers = {name.lower(): value for name, value in headers.items()} + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except ValueError: + self.json = None + + @property + def instance(self): + """X-Instance-Id: какая реплика API ответила (ID её контейнера).""" + return self.headers.get("x-instance-id") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30): + """HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError. + + Заодно проверяет то, что контракт требует от любого ответа: + X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401. + """ + all_headers = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + if data is not None: + all_headers.setdefault("Content-Type", "application/json") + if token is not None: + all_headers["Authorization"] = "Bearer " + token + + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + response = Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + response = Response(error.code, error.headers, error.read()) + + expected = expected if isinstance(expected, tuple) else (expected,) + check(response.status in expected, + f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}") + request_id = response.headers.get("x-request-id") + check(request_id, f"{method} {path}: no X-Request-Id header") + check(response.instance, f"{method} {path}: no X-Instance-Id header") + if response.status >= 400: + error = (response.json or {}).get("error") or {} + check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format") + check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header") + if response.status == 401: + check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate") + return response + + +def try_call(method, path, *, body=None, token=None, headers=None, timeout=2): + """Запрос без проверок и без исключений — для нагрузки во время отказа. + + Возвращает Response или None, если соединение оборвалось / вышел timeout. + """ + all_headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + all_headers["Content-Type"] = "application/json" + if token is not None: + all_headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + return Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + try: + return Response(error.code, error.headers, error.read()) + except OSError: + return None + except (urllib.error.URLError, OSError): + return None + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8.""" + return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + + +def login(username, password, expected=201): + return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)}) + + +def new_user(prefix="user", password=PASSWORD): + """Регистрирует пользователя и входит. Возвращает dict: id, username, password, token.""" + username = f"{prefix}_{uuid.uuid4().hex[:10]}" + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": prefix.title(), "password": password}).json + session = login(username, password).json + return {**user, "password": password, "token": session["token"]} + + +def new_chat(owner, *others, title="Test chat"): + member_ids = [owner["id"]] + [user["id"] for user in others] + return call("POST", "/api/v1/chats", 201, token=owner["token"], + body={"title": title, "member_ids": member_ids}).json + + +def new_key(): + return uuid.uuid4().hex + + +def send(user, chat_id, text, expected=201, key=None): + """POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body={"text": text}, headers={"Idempotency-Key": key or new_key()}) + + +def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None): + """POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся.""" + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body=body, headers={"Idempotency-Key": key or new_key()}) + + +def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=201, timeout=60): + """POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра.""" + query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="") + headers = {"Content-Type": content_type} if content_type is not None else {} + return call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data, + token=user["token"] if user else None, headers=headers, timeout=timeout) + + +def read_history(user, chat_id, limit=100): + """Вся история чата по страницам. Возвращает (сообщения, число_страниц).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "") + page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json + items += page["items"] + pages += 1 + cursor = page["next_cursor"] + if cursor is None: + return items, pages + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while True: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply: + if reply.status == 200: + return + except (urllib.error.URLError, OSError): + pass + check(time.monotonic() < deadline, f"API is not ready after {seconds} s") + time.sleep(0.5) + + +def compose(*args, env=None): + """docker compose в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам.""" + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def compose_output(*args, env=None): + """docker compose и его вывод (для logs, ps).""" + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env) + return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace") + + +def docker(*args, check_exit=True): + """docker , возвращает stdout.""" + result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit) + return result.stdout.decode("utf-8", "replace").strip() + + +def api_containers(include_stopped=False): + """Полные ID контейнеров сервиса api (по умолчанию только запущенные).""" + args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"] + return sorted(compose_output(*args).split()) + + +def health(container): + """Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''.""" + return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container, + check_exit=False) + + +def lb_servers(): + """Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}.""" + with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply: + rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace")))) + header = [name.lstrip("# ") for name in rows[0]] + servers = {} + for row in rows[1:]: + item = dict(zip(header, row)) + if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"): + servers[item["svname"]] = item.get("status", "") + return servers + + +def lb_up_count(): + """Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP).""" + return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP")) + + +def seen_instances(token, requests=20): + """Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me.""" + return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)} + + +def wait_balanced(replicas, seconds=60): + """Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке.""" + deadline = time.monotonic() + seconds + while True: + running = api_containers() + healthy = [container for container in running if health(container) == "healthy"] + try: + up = lb_up_count() + except OSError: + up = 0 + if len(healthy) >= replicas and up >= replicas: + return running + check(time.monotonic() < deadline, + f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}") + time.sleep(0.5) + + +def redis_cli(*args): + """redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод.""" + command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args) + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command], + cwd=ROOT, capture_output=True, check=True) + return result.stdout.decode("utf-8", "replace").strip() + + +def session_key(token): + """Ключ сессии в Redis, как его считает auth.py: session:.""" + return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def psql(sql): + """Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки). + + Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя. + """ + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, check=False, + ) + output = result.stdout.decode("utf-8", "replace").strip() + error = result.stderr.decode("utf-8", "replace").strip() + return result.returncode, output, error + + +def app_python(code, *args, service="api", env=None): + """Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API). + + Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON). + """ + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return output + + +def janitor(*args, env=None): + """Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт.""" + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += ["janitor", "python", "janitor.py", "--once", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return json.loads(output.splitlines()[-1]) + + +def sql_value(query): + """Одно значение из PostgreSQL (psql -A -t).""" + code, output, error = psql(query) + check(code == 0, f"psql failed: {error[-300:]}") + return output + + +def run_tests(tests): + """Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал.""" + failed = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста + failed += 1 + print(f"FAIL: {test.__name__}: {exc}") + print(f"{len(tests) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/demo_instances.py b/student-tests/demo_instances.py new file mode 100644 index 0000000..8f97f1c --- /dev/null +++ b/student-tests/demo_instances.py @@ -0,0 +1,16 @@ + +from common import call, compose_output, new_user + + +def main(): + user = new_user("demo") + print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:") + for number in range(1, 9): + reply = call("GET", "/api/v1/users/me", 200, token=user["token"]) + print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}") + print() + print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 0000000..81416de --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,28 @@ + +import subprocess + +from common import ROOT + +OUTPUT = ROOT / "evidence" / "explain-history.txt" + + +def main(): + sql = (ROOT / "student-tests" / "explain.sql").read_bytes() + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + cwd=ROOT, input=sql, capture_output=True, check=False, + ) + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_bytes(result.stdout + result.stderr) + if result.returncode != 0: + print(result.stderr.decode("utf-8", "replace")) + raise SystemExit("FAIL: psql returned an error, see output above") + for line in result.stdout.decode("utf-8", "replace").splitlines(): + if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line: + print(line) + print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 0000000..01cd9e7 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,76 @@ + +\pset footer off +BEGIN; + +CREATE TEMP TABLE seed_chats AS + SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size + FROM generate_series(0, 50) AS n; +CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; + +INSERT INTO users (id, username, display_name, password_hash) + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; +INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), + timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' + FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; +ANALYZE messages; + +SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset +SELECT created_at AS cursor_time, id AS cursor_id FROM messages + WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== Data volume' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat +FROM messages; + +\echo '' +\echo '=== History index' +SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history'; + +\echo '' +\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY m.created_at, m.id LIMIT 51; + +\echo '' +\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id OFFSET 10000 LIMIT 51; + +\echo '' +\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)' +DROP INDEX messages_history; +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; +\echo '' +\echo '=== ROLLBACK: test data removed, index is back' +SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%'; diff --git a/student-tests/failover_metrics.py b/student-tests/failover_metrics.py new file mode 100644 index 0000000..19421a1 --- /dev/null +++ b/student-tests/failover_metrics.py @@ -0,0 +1,129 @@ +import json +import platform +import sys +import threading +import time + +from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced +from loadgen import Load, summary + +WARMUP_SECONDS = 3 +AFTER_SECONDS = 10 +OUTPUT = ROOT / "evidence" / "failover-metrics.json" + + +class LbWatch: + """Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP).""" + + def __init__(self): + self.points, self.stopping = [], threading.Event() + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stopping.is_set(): + try: + self.points.append((time.monotonic(), lb_up_count())) + except OSError: + pass + time.sleep(0.1) + + def start(self): + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join(timeout=3) + + def first_drop(self, after, replicas): + return next((t for t, up in self.points if t >= after and up < replicas), None) + + +def measure(container, mode, replicas): + wait_balanced(replicas) + load, watch = Load(), LbWatch() + watch.start() + load.start() + time.sleep(WARMUP_SECONDS) + t0 = time.monotonic() + if mode == "stop": + docker("stop", "--time", "8", container) + else: + docker("kill", container) + command_seconds = time.monotonic() - t0 + time.sleep(max(0.0, AFTER_SECONDS - command_seconds)) + end = t0 + AFTER_SECONDS + load.stop() + watch.stop() + + errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]] + errors_end = max((e[0] + e[5] for e in errors), default=None) + drop = watch.first_drop(t0, replicas) + served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]] + target_hits = sum(1 for instance in served_after if instance and container.startswith(instance)) + integrity = load.verify() + + started = time.monotonic() + docker("start", container) + wait_balanced(replicas) + rejoin_seconds = time.monotonic() - started + + return { + "replica": container[:12], + "mode": mode, + "command_seconds": round(command_seconds, 2), + "lb_excluded_after_seconds": round(drop - t0, 2) if drop else None, + "errors": len(errors), + "error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"], + "errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0, + "before": summary(load, t0 - WARMUP_SECONDS, t0), + "after": summary(load, t0, end), + "stopped_replica_answers_after_stop": target_hits, + "integrity": integrity, + "rejoin_seconds": round(rejoin_seconds, 2), + } + + +def main(): + modes = sys.argv[1:] or ["stop", "kill"] + check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]") + containers = api_containers() + replicas = len(containers) + check(replicas >= 2, "need at least 2 running api replicas") + print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, " + f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and " + f"{AFTER_SECONDS} s after the failure") + runs = [] + for container in containers: + for mode in modes: + print(f"--- {mode} {container[:12]} ...") + run = measure(container, mode, replicas) + runs.append(run) + print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after " + f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, " + f"stopped after {run['errors_stopped_after_seconds']} s") + print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; " + f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; " + f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}") + print(f" confirmed {run['integrity']['confirmed_writes']}, in history " + f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; " + f"back in the balancer {run['rejoin_seconds']} s after docker start") + check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered") + check(run["after"]["write_rps"] > 0, "no successful writes after the failure") + print() + print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s") + for r in runs: + print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} " + f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} " + f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS, + "after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()}, + "runs": runs, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; " + f"saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/progressive.jpg b/student-tests/fixtures/progressive.jpg new file mode 100644 index 0000000..38faa2b Binary files /dev/null and b/student-tests/fixtures/progressive.jpg differ diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000..6816ff5 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 0000000..a83b5de --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,74 @@ +"""Синтетические изображения для тестов вложений: только стандартная библиотека. + +PNG собирается вручную (сигнатура, IHDR, IDAT, IEND), поэтому можно получить любой размер +в пикселях и байтах, а также испортить конкретный байт. JPEG берётся из fixtures/. +""" +import os +import struct +import zlib +from pathlib import Path + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +ROOT = Path(__file__).resolve().parents[1] +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) + + +def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes: + """PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой). + + pad_to — дополнить файл служебным чанком prVt до точного размера в байтах. + idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое). + """ + channels = 1 if gray else 3 + header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace) + if idat is None: + compressor = zlib.compressobj(6) + parts = [] + # Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход. + passes = [(width, height)] if not interlace else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + for pass_width, pass_height in passes: + if pass_width <= 0 or pass_height <= 0: + continue + row = b"\x00" + bytes(pass_width * channels) + for _ in range(pass_height): + parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row)) + parts.append(compressor.flush()) + idat = b"".join(parts) + body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat) + if pad_to is not None: + need = pad_to - len(body) - 12 - 12 # чанк-заполнитель + IEND + assert need >= 0, "image is already larger than pad_to" + body += chunk(b"prVt", bytes(need)) + return body + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями).""" + side = int(((target_bytes - 4096) / 3) ** 0.5) + return png(side, side, noise=True) + + +def sample_png() -> bytes: + return (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (FIXTURES / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (FIXTURES / "progressive.jpg").read_bytes() + + +def flip_byte(data: bytes, position: int) -> bytes: + return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:] + + +GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 0000000..998548c --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,102 @@ +import math +import threading +import time + +from common import check, new_chat, new_key, new_user, read_history, try_call + + +class Load: + def __init__(self, writers=4, timeout=2): + self.alice, self.bob = new_user("load_a"), new_user("load_b") + self.chat = new_chat(self.alice, self.bob, title="load") + self.writers, self.timeout = writers, timeout + self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с) + self.confirmed = [] # сообщения, на которые пришёл 201 + self.lock = threading.Lock() + self.stopping = threading.Event() + self.threads = [] + + def _record(self, kind, started, reply, ok): + code = reply.status if reply is not None else "conn" + instance = reply.instance if reply is not None else None + with self.lock: + self.events.append((started, kind, ok, code, instance, time.monotonic() - started)) + + def _writer(self, number): + path = f"/api/v1/chats/{self.chat['id']}/messages" + sent = 0 + key, text = new_key(), f"load-{number}-{sent}" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("POST", path, body={"text": text}, token=self.alice["token"], + headers={"Idempotency-Key": key}, timeout=self.timeout) + ok = reply is not None and reply.status == 201 + self._record("write", started, reply, ok) + if ok: + with self.lock: + self.confirmed.append(reply.json) + sent += 1 + key, text = new_key(), f"load-{number}-{sent}" + else: + time.sleep(0.05) # и повторить тот же text с тем же ключом + + def _reader(self): + path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout) + self._record("read", started, reply, reply is not None and reply.status == 200) + time.sleep(0.05) + + def start(self): + self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)] + self.threads.append(threading.Thread(target=self._reader, daemon=True)) + for thread in self.threads: + thread.start() + + def stop(self): + self.stopping.set() + for thread in self.threads: + thread.join(timeout=self.timeout + 5) + + def window(self, start, end, kind=None): + with self.lock: + return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)] + + def verify(self): + """Все подтверждённые сообщения есть в истории, id и тексты не повторяются.""" + history, _ = read_history(self.bob, self.chat["id"]) + ids = [message["id"] for message in history] + texts = [message["text"] for message in history] + by_id = {message["id"]: message for message in history} + check(len(ids) == len(set(ids)), "duplicated message ids in history") + check(len(texts) == len(set(texts)), "a retried request created a duplicate message") + lost = [m for m in self.confirmed if by_id.get(m["id"]) != m] + check(not lost, f"{len(lost)} confirmed messages are lost or changed") + return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0} + + +def summary(load, start, end): + """Успешные RPS, ошибки и задержки на отрезке [start, end).""" + events = load.window(start, end) + writes = [e for e in events if e[1] == "write"] + ok_writes = sorted(e[5] for e in writes if e[2]) + errors = [e for e in events if not e[2]] + seconds = max(end - start, 1e-6) + + def percentile(p): + return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None + + codes = {} + for e in errors: + codes[str(e[3])] = codes.get(str(e[3]), 0) + 1 + return { + "seconds": round(seconds, 2), + "write_rps": round(len(ok_writes) / seconds, 1), + "reads": len([e for e in events if e[1] == "read"]), + "errors": len(errors), + "error_codes": codes, + "write_p50_ms": percentile(0.50), + "write_p95_ms": percentile(0.95), + "write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None, + } diff --git a/student-tests/redis_info.py b/student-tests/redis_info.py new file mode 100644 index 0000000..6535eae --- /dev/null +++ b/student-tests/redis_info.py @@ -0,0 +1,29 @@ +import subprocess + +from common import ROOT, login, new_user, redis_cli, session_key + + +def without_password(): + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"], + cwd=ROOT, capture_output=True, check=False) + return (result.stdout + result.stderr).decode("utf-8", "replace").strip() + + +def main(): + user = new_user("info") + token = login(user["username"], user["password"]).json["token"] + key = session_key(token) + print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}") + print(f"TTL of the key, seconds: {redis_cli('ttl', key)}") + keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines() + print(f"sessions in Redis: {len(keys)}") + for name in ("appendonly", "appendfsync", "maxmemory-policy"): + print("config", " = ".join(redis_cli("config", "get", name).split())) + for line in redis_cli("info", "persistence").splitlines(): + if line.startswith(("aof_enabled", "aof_last_write_status")): + print(line) + print(f"without password: {without_password()}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 0000000..f2f3ffb --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,88 @@ +import json +import os +import time + +from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced +from loadgen import Load, summary + +OUTPUT = ROOT / "evidence" / "rolling-update.json" + + +def env_of(container): + lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +def wait_new_replica(container, up_needed, seconds=90): + deadline = time.monotonic() + seconds + while health(container) != "healthy" or lb_up_count() < up_needed: + check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") + time.sleep(0.2) + + +def main(): + old = api_containers() + replicas = len(old) + check(replicas >= 2, "need at least 2 running api replicas") + wait_balanced(replicas) + old_version = env_of(old[0]).get("APP_VERSION") + version = time.strftime("rolling-%Y%m%d-%H%M%S") + env = dict(os.environ, APP_VERSION=version) + + print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") + started = time.monotonic() + compose("build", "api", "migrate", env=env) + print(f"image built in {time.monotonic() - started:.1f} s") + migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) + print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] + .split("messenger.db: ")[-1]) + + load = Load() + load.start() + time.sleep(3) + t0 = time.monotonic() + steps = [] + for container in old: + step_start = time.monotonic() + before = set(api_containers()) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) + new = sorted(set(api_containers()) - before) + check(len(new) == 1, f"expected one new container, got {len(new)}") + wait_new_replica(new[0], replicas + 1) + ready = time.monotonic() - step_start + docker("stop", "--time", "15", container) + docker("rm", container) + steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), + "step_seconds": round(time.monotonic() - step_start, 2)}) + print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " + f"step {steps[-1]['step_seconds']} s") + t1 = time.monotonic() + time.sleep(3) + load.stop() + + final = api_containers() + versions = sorted({env_of(c).get("APP_VERSION") for c in final}) + integrity = load.verify() + during = summary(load, t0, t1) + before_stats = summary(load, t0 - 3, t0) + print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") + print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " + f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " + f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") + print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " + f"lost 0, duplicates 0") + check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") + check(versions == [version], "not every replica runs the new version") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "from_version": old_version, "to_version": version, "replicas": replicas, + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, + "steps": steps, "update_seconds": round(t1 - t0, 2), + "before": before_stats, "during": during, "integrity": integrity, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: all {replicas} replicas replaced one by one under load, " + f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_failover.py b/student-tests/run_failover.py new file mode 100644 index 0000000..0f28ec8 --- /dev/null +++ b/student-tests/run_failover.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys + +from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced + +HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + + +def ps(): + return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() + + +def aim(wanted): + """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" + probe = call("GET", "/health/live", 200).instance + if not any(container.startswith(probe) for container in wanted): + call("GET", "/health/live", 200) # теперь последней ответила другая реплика + + +def main(): + containers = api_containers() + check(len(containers) >= 2, "need at least 2 running api replicas") + print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") + stopped_once = set() + for attempt in range(1, 7): + wait_balanced(len(containers)) + if len(containers) == 2: + aim(set(containers) - stopped_once) + print(f"=== run {attempt}: api containers before") + print(ps()) + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public failover test failed") + print("api containers after") + print(ps()) + stopped = [container for container in containers if container not in api_containers()] + check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") + print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " + f"other api containers kept running, postgres/redis/lb were not touched") + stopped_once.add(stopped[0]) + docker("start", stopped[0]) + wait_balanced(len(containers)) + print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") + if stopped_once == set(containers): + break + check(stopped_once == set(containers), "not every replica was stopped by the public test") + print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_persistence.py b/student-tests/run_persistence.py new file mode 100644 index 0000000..66b3293 --- /dev/null +++ b/student-tests/run_persistence.py @@ -0,0 +1,47 @@ +import os +import subprocess +import sys +import time + +from common import BASE_URL, ROOT, check, compose_output, docker + +HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + + +def started(): + containers = compose_output("ps", "-q").split() + lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines() + return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)} + + +def main(): + before = started() + print("before: container, ID, started at") + for name, (cid, at) in sorted(before.items()): + print(f" {name:<28} {cid[:12]} {at}") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + begin = time.monotonic() + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + seconds = time.monotonic() - begin + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public persistence test failed") + after = started() + print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?") + restarted = [] + for name, (cid, at) in sorted(after.items()): + same_id = before.get(name, ("", ""))[0] == cid + new_start = before.get(name, ("", ""))[1] != at + print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}") + check(same_id, f"{name}: container was recreated, volumes/IDs must stay") + if new_start: + restarted.append(name.split("-")[-2]) + for service in ("postgres", "redis", "s3", "api", "janitor", "lb"): + check(service in restarted, f"{service} was not restarted by the hook") + print("PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, " + "new start time); data, the attachment bytes and the old token survived") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..7c770d9 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,258 @@ +import base64 +import datetime as dt +import uuid + +from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send + + +def parse_time(value): + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + + +def test_duplicate_username(): + body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} + call("POST", "/api/v1/users", 201, body=body) + call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user")) + print("PASS: duplicate username returns 409") + + +def test_invalid_json(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", "/api/v1/users", 400, raw=b'{"username":') + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json", + headers={"Idempotency-Key": new_key()}) + print("PASS: invalid JSON returns 400") + + +def test_wrong_fields(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD}) + call("POST", "/api/v1/users", 400, + body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True}) + # Отправителя нельзя подставить: он берётся из сессии. + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()}) + print("PASS: wrong and extra fields return 400") + + +def test_foreign_chat(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + chat = new_chat(alice, bob) + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"]) + send(eve, chat["id"], "not my chat", expected=404) + # Несуществующий чат выглядит так же, как чужой. + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"]) + print("PASS: foreign chat returns 404") + + +def test_empty_message(): + alice = new_user("alice") + chat = new_chat(alice) + for text in ("", " ", "\n\t"): + send(alice, chat["id"], text, expected=400) + send(alice, chat["id"], "я" * 4001, expected=400) + send(alice, chat["id"], "я" * 4000) + print("PASS: empty and too long messages return 400, 4000 chars is ok") + + +def test_pagination(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)] + + history, pages = read_history(bob, chat["id"], limit=2) + check(pages == 4, f"expected 4 pages, got {pages}") + check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages") + + other = new_chat(alice) + for i in range(3): + send(alice, other["id"], f"Other {i}") + first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) + print("PASS: pagination works across multiple pages, foreign cursor returns 400") + + +def test_empty_cursor(): + alice = new_user("alice") + chat = new_chat(alice) + send(alice, chat["id"], "one message") + for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"): + for query in ("?cursor=", "?cursor", "?limit=10&cursor="): + call("GET", path + query, 400, token=alice["token"]) + call("GET", path, 200, token=alice["token"]) + print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page") + + +def test_chat_list_pagination(): + alice, bob = new_user("alice"), new_user("bob") + created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] + seen, cursor = [], None + while True: + query = "?limit=1" + (f"&cursor={cursor}" if cursor else "") + page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json + seen += [chat["id"] for chat in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + check(seen == created, "chat list pages are wrong") + print("PASS: chat list pagination") + + +def test_unicode(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob, title="Чат 👋") + text = "Привет 👋 你好" + check(send(alice, chat["id"], text).json["text"] == text, "text changed on write") + history, _ = read_history(bob, chat["id"]) + check(history[-1]["text"] == text, "text changed on read") + print("PASS: unicode text is stored as is") + + +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}, + headers={"Idempotency-Key": new_key()}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + + +def test_user_has_no_password(): + body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} + user = call("POST", "/api/v1/users", 201, body=body).json + check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}") + print("PASS: user response has no password or hash") + + +def test_password_rules(): + def register(password, expected): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password} + call("POST", "/api/v1/users", expected, body=body) + + register("x" * 11, 400) + register("x" * 129, 400) + register(" " * 12, 400) + register("x" * 12, 201) + register("я" * 128, 201) + print("PASS: password length 12..128, blank password is rejected") + + +def test_login_gives_token(): + username = "lg_" + uuid.uuid4().hex[:10] + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Login", "password": PASSWORD}).json + reply = login(username, PASSWORD) + session = reply.json + check(session["token_type"] == "Bearer", "token_type must be Bearer") + check(session["user"] == user, "session user differs from registered user") + check(len(session["token"]) >= 43, "token is too short") + check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque") + check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached") + left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s") + me = call("GET", "/api/v1/users/me", token=session["token"]).json + check(me == user, "/users/me returned another user") + print("PASS: Basic login returns opaque Bearer token with expiry") + + +def test_wrong_password(): + user = new_user("wp") + wrong = login(user["username"], "Wrong-password-000", expected=401) + unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) + check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") + check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") + print("PASS: wrong password and unknown user both return the same 401") + + +def test_malformed_basic(): + no_colon = base64.b64encode(b"just_username").decode() + for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"): + headers = {} if header is None else {"Authorization": header} + call("POST", "/api/v1/auth/sessions", 401, headers=headers) + print("PASS: malformed Basic header returns 401") + + +def test_password_with_colon_and_unicode(): + password = "пароль:с:двоеточием 👋" + user = new_user("colon", password=password) + call("GET", "/api/v1/users/me", token=user["token"]) + print("PASS: password with colon and unicode works") + + +def test_bearer_required(): + user = new_user("br") + for headers in ({}, {"Authorization": basic(user["username"], user["password"])}, + {"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}): + reply = call("GET", "/api/v1/users/me", 401, headers=headers) + check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer") + call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]}) + print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401") + + +def test_x_user_id_ignored(): + alice, bob = new_user("alice"), new_user("bob") + me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json + check(me["id"] == alice["id"], "X-User-Id changed the current user") + print("PASS: X-User-Id does not change the current user") + + +def test_auth_before_body(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) + print("PASS: 401 comes before 400 and 404") + + +def test_logout(): + user = new_user("lo") + second = login(user["username"], user["password"]).json["token"] + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + call("GET", "/api/v1/users/me", 401, token=user["token"]) + call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива + again = login(user["username"], user["password"]).json["token"] + check(again != user["token"], "new login must give a new token") + call("GET", "/api/v1/users/me", 200, token=again) + print("PASS: logout revokes only the current session") + + +if __name__ == "__main__": + run_tests([ + test_duplicate_username, + test_invalid_json, + test_wrong_fields, + test_foreign_chat, + test_empty_message, + test_pagination, + test_empty_cursor, + test_chat_list_pagination, + test_unicode, + test_nul_character_is_stored, + test_user_has_no_password, + test_password_rules, + test_login_gives_token, + test_wrong_password, + test_malformed_basic, + test_password_with_colon_and_unicode, + test_bearer_required, + test_x_user_id_ignored, + test_auth_before_body, + test_logout, + ]) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..e1178c1 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,268 @@ +"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ. + +Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа: +пользовательская ошибка не должна становиться 500. +""" +import hashlib +import http.client +import socket +import urllib.parse +import uuid +import zlib + +import images +from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload + +URL = urllib.parse.urlsplit(BASE_URL) + + +def setup(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + return alice, bob, eve, new_chat(alice, bob) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def check_error(reply, code): + check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}") + + +def test_png_and_jpeg_roundtrip(): + alice, bob, _, chat = setup() + for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"), + (images.sample_jpeg(), "image/jpeg", "sample.jpg"), + (images.progressive_jpeg(), "image/jpeg", "progressive.jpg"), + (images.png(33, 17, interlace=1), "image/png", "interlaced.png"), + (images.png(1, 1, gray=True), "image/png", "1x1.png")): + att = upload(alice, chat["id"], data, ctype, name).json + check(att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None, "not ready") + check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ") + check(att["content_type"] == ctype and att["filename"] == name, "metadata differs") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta == att, "GET metadata differs from POST response") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "downloaded bytes differ") + check(got.headers["content-type"] == ctype, "wrong Content-Type on download") + check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length") + check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing") + print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes") + print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte") + + +def test_size_limit_exact_boundary(): + alice, _, _, chat = setup() + exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES) + check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB") + att = upload(alice, chat["id"], exact).json + check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed") + print(f"exactly {len(exact)} bytes (10 MiB) -> 201") + over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1) + reply = upload(alice, chat["id"], over, expected=413) + check_error(reply, "payload_too_large") + print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body") + print("PASS: the 10 MiB limit is exact") + + +def chunked_upload(user, chat_id, total, piece=64 * 1024): + """Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении.""" + conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60) + + def body(): + sent = 0 + head = images.sample_png()[:64] + yield head + sent += len(head) + while sent < total: + part = bytes(min(piece, total - sent)) + sent += len(part) + yield part + try: + conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(), + headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]}, + encode_chunked=True) + reply = conn.getresponse() + return reply.status, reply.read() + finally: + conn.close() + + +def test_size_limit_streaming(): + alice, _, _, chat = setup() + status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024) + check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}") + print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed") + call("GET", "/health/ready", 200) + print("PASS: oversized streamed body is cut off, the API keeps working") + + +def test_pixel_limit(): + alice, _, _, chat = setup() + at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей + att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json + print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 201 {att['id'][:8]}") + over = images.png(8001, 5000, gray=True) + check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large") + print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413") + bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01") + check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large") + print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded") + print("PASS: pixel limit is checked before decompression") + + +def test_damaged_images(): + alice, _, _, chat = setup() + png, jpeg = images.sample_png(), images.sample_jpeg() + cases = [ + ("PNG cut in half", png[:len(png) // 2], "image/png"), + ("PNG without IEND", png[:-12], "image/png"), + ("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"), + ("PNG with garbage after IEND", png + b"garbage", "image/png"), + ("PNG signature only", png[:8], "image/png"), + ("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"), + ("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"), + ("JPEG header only", jpeg[:20], "image/jpeg"), + ("random bytes", uuid.uuid4().bytes * 100, "image/png"), + ("plain text", "это не картинка".encode(), "image/jpeg"), + ] + for name, data, ctype in cases: + reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415) + check_error(reply, "unsupported_media_type") + print(f"{name} ({len(data)} bytes, {ctype}) -> 415") + check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request") + print("empty body -> 400") + print("PASS: damaged and foreign files are rejected with 415, never 500") + + +def test_mime_substitution(): + alice, _, _, chat = setup() + cases = [ + ("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"), + ("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"), + ("GIF bytes sent as image/png", images.GIF, "image/png"), + ("PNG sent as image/gif", images.sample_png(), "image/gif"), + ("PNG sent as text/html", images.sample_png(), "text/html"), + ("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"), + ("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"), + ] + for name, data, ctype in cases: + check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type") + print(f"{name} -> 415") + att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json + check(att["content_type"] == "image/png", "content type must be normalised") + print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png") + print("PASS: the type is taken from the content; a declared type that does not match is refused") + + +def test_filenames(): + alice, _, _, chat = setup() + bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png", + "bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256] + for name in bad: + check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request") + print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400") + check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request") + print("no filename -> 400") + for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"): + att = upload(alice, chat["id"], images.sample_png(), filename=name).json + check(att["filename"] == name, "display name changed") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded") + print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only") + print("PASS: filename never becomes a path; dangerous names are refused") + + +def test_access_control(): + alice, bob, eve, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + for suffix in ("", "/content"): + reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + check_error(reply, "not_found") + call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400") + check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found") + check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found") + upload(None, chat["id"], images.sample_png(), expected=401) + print("upload into someone else's / unknown chat -> 404, without token -> 401") + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", 400, token=bob["token"]) + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=original", 200, token=bob["token"]) + print("variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200") + print("PASS: only chat members can see an attachment") + + +def test_message_attachments(): + alice, bob, eve, chat = setup() + first = upload(alice, chat["id"], images.sample_png()).json + second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json + msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json + check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed") + print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member") + plain = post_message(bob, chat["id"], "без вложений").json + check(plain["attachment_ids"] == [], "default attachment_ids must be []") + reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json + history, _ = read_history(bob, chat["id"]) + check(history == [msg, plain, reused], "history differs from the confirmed messages") + print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed") + + other_chat = new_chat(alice, eve) + foreign = upload(alice, other_chat["id"], images.sample_png()).json + for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"), + ([first["id"], foreign["id"]], "own + foreign attachment"), + ([str(uuid.uuid4())], "unknown attachment_id")): + check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found") + print(f"{what} -> 404") + check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found") + print("outsider referencing an attachment of the chat -> 404") + for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"), + ([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"), + (["not-a-uuid"], "x", "bad UUID"), + ([], "", "empty text without attachments"), + ([], " ", "whitespace text without attachments")): + check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request") + print(f"{what} -> 400") + history_after, _ = read_history(bob, chat["id"]) + check(history_after == history, "a rejected message was stored") + print("PASS: only attachments of the same chat can be linked; rejected requests change nothing") + + +def test_idempotency_with_attachments(): + alice, bob, _, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json + key = new_key() + first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + check(again == first, "replay returned a different message") + check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict") + check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict") + key2 = new_key() + plain = post_message(alice, chat["id"], "текст", None, key=key2).json + check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field") + history, _ = read_history(bob, chat["id"]) + check(len(history) == 2, "replays created extra messages") + print("same key + same attachments -> same 201; other attachments -> 409; missing field == []") + print("PASS: Idempotency-Key covers attachment_ids") + + +def test_aborted_upload(): + alice, _, _, chat = setup() + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000)) + sock.close() # клиент пропал на середине тела + print("sent 1.1 MB of a declared 5 MB body and closed the connection") + att = upload(alice, chat["id"], images.sample_png()).json + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + print("the API keeps serving uploads; nothing of the aborted request is visible " + "(DB/S3 side is checked by test_consistency.py)") + print("PASS: an interrupted upload is dropped") + + +if __name__ == "__main__": + run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming, + test_pixel_limit, test_damaged_images, test_mime_substitution, test_filenames, test_access_control, + test_message_attachments, test_idempotency_with_attachments, test_aborted_upload]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 0000000..01f2251 --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,103 @@ +from collections import Counter + +import images +from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload + + +def test_requests_spread_over_replicas(): + alice = new_user("alice") + replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)] + served = Counter(reply.instance for reply in replies) + print(f"20 x GET /users/me with one token -> replicas: {dict(served)}") + check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned") + check(len(served) >= 2, "all requests went to one replica: balancing does not work") + print("PASS: one session works on every replica, no second login (sessions are shared in Redis)") + + +def test_write_on_one_read_on_other(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = send(alice, chat["id"], "written on one replica") + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]) + check(sent.json in reply.json["items"], "confirmed message is not visible on another replica") + readers.add(reply.instance) + print(f"message written by {sent.instance}; read back by {sorted(readers)}") + check(readers - {sent.instance}, "no read went to another replica") + print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)") + + +def test_logout_seen_by_all_replicas(): + user = new_user("logout") + out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)] + print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}") + check(len(set(after)) >= 2, "only one replica answered") + print("PASS: logout on one replica revokes the session on all of them") + + +def test_idempotent_retry_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "retry me", key=key) + for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику + retry = send(alice, chat["id"], "retry me", key=key) + if retry.instance != first.instance: + break + print(f"first POST on {first.instance} -> 201 {first.json['id']}; " + f"same key on {retry.instance} -> {retry.status} {retry.json['id']}") + check(retry.instance != first.instance, "retry never reached another replica") + check(retry.json == first.json, "retry on another replica returned a different message") + items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"] + check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate") + print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message") + + +def test_attachment_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + sent = upload(alice, chat["id"], data) + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"]) + check(reply.data == data, "another replica returned different bytes") + readers.add(reply.instance) + print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}") + check(readers - {sent.instance}, "no download went to another replica") + print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)") + + +def test_instance_header_everywhere(): + containers = api_containers() + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + replies = [ + call("GET", "/health/live", 200), + call("GET", "/health/ready", 200), + send(alice, chat["id"], "hello"), + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}), + call("GET", "/api/v1/users/me", 401), + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]), + call("POST", "/api/v1/users", 409, + body={"username": alice["username"], "display_name": "x", "password": alice["password"]}), + ] + for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер + check(any(container.startswith(reply.instance) for container in containers), + f"X-Instance-Id {reply.instance} is not a running api container") + print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container " + f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}") + print("PASS: X-Instance-Id is in every response and points to a real container") + + +if __name__ == "__main__": + run_tests([ + test_requests_spread_over_replicas, + test_write_on_one_read_on_other, + test_logout_seen_by_all_replicas, + test_idempotent_retry_on_other_replica, + test_attachment_on_other_replica, + test_instance_header_everywhere, + ]) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 0000000..af10ad1 --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,275 @@ +"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка). + +Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`), +выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает +janitor вручную. Данные синтетические, volumes не удаляются. + + python student-tests/test_consistency.py [s3|crash|orphan|aborted|all] +""" +import json +import socket +import sys +import time +import urllib.parse + +import images +from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat, + new_user, post_message, run_tests, sql_value, upload, wait_ready) + +URL = urllib.parse.urlsplit(BASE_URL) + +# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но finish_upload +# (UPDATE ... state='ready' + COMMIT) падает — как если бы реплика умерла сразу после PUT в S3. +CRASH_AFTER_PUT = r''' +import json, struct, sys, tempfile, zlib +from uuid import UUID +import attachments, db, storage, uploads +chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2]) +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b"")) +f = tempfile.SpooledTemporaryFile(); f.write(data) +import hashlib +upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest()) +image = uploads.inspect(upload, "image/png") +created = {} +real_intent = db.create_upload_intent +def remember(attachment_id, *args): + created["id"], created["key"] = str(attachment_id), args[-1] + return real_intent(attachment_id, *args) +def crash(attachment_id): + raise RuntimeError("simulated crash between S3 PUT and DB COMMIT") +db.create_upload_intent = remember +db.finish_upload = crash +db.pool.open() +try: + attachments.store(chat_id, owner_id, "crash.png", upload, image) + outcome = "stored" +except RuntimeError as exc: + outcome = str(exc) +print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"], + "object_exists": storage.exists(created["key"])})) +''' + +OBJECT_EXISTS = r''' +import json, sys, storage +print(json.dumps({"exists": storage.exists(sys.argv[1])})) +''' + +PUT_ORPHAN = r''' +import hashlib, io, json, uuid, storage +data = b"orphan object without a row in PostgreSQL" +key = storage.new_key(uuid.uuid4().hex) +storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest()) +print(json.dumps({"key": key})) +''' + +DELETE_OBJECT = r''' +import json, sys, storage +storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]})) +''' + +COUNT_OBJECTS = r''' +import json, storage +print(json.dumps({"objects": sum(1 for _ in storage.list_objects())})) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def object_exists(key): + return last_json(app_python(OBJECT_EXISTS, key))["exists"] + + +def backdate(where): + sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}") + + +def uploading_rows(chat_id): + return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'")) + + +def wait_download(user, att, seconds=60): + deadline = time.monotonic() + seconds + while True: + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"]) + if reply.status == 200: + return reply + check(time.monotonic() < deadline, "download did not recover after S3 start") + time.sleep(1) + + +def s3_outage(stop_action, start_action, label): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + att = upload(alice, chat["id"], data).json + before = uploading_rows(chat["id"]) + print(f"--- S3 {label}: docker compose {stop_action} s3") + compose(stop_action, "s3") + try: + started = time.monotonic() + reply = upload(alice, chat["id"], data, expected=503) + ms = (time.monotonic() - started) * 1000 + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + body = reply.data.decode().lower() + check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")), + "error leaks storage details") + print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body") + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + print("download of an existing file -> 503") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta["sha256"] == att["sha256"], "metadata changed") + post_message(alice, chat["id"], "текст без файла") + post_message(alice, chat["id"], "", [att["id"]]) + call("GET", "/health/ready", 200) + print("metadata, messages (also with an already stored attachment) and /health/ready keep working: " + "S3 is needed only for file bytes") + after = uploading_rows(chat["id"]) + print(f"uploading rows of this chat: before {before}, after the failed upload {after} " + "(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API") + finally: + print(f"docker compose {start_action} s3") + compose(start_action, "s3") + got = wait_download(bob, att) + check(got.data == data, "file changed after S3 recovery") + print("after S3 recovery the old file downloads byte for byte") + upload(alice, chat["id"], data) + print("new upload -> 201") + leftover = uploading_rows(chat["id"]) + if leftover: + backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'") + report = janitor() + print(f"janitor after grace period: {json.dumps(report)}") + check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain") + print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up") + + +def test_s3_stopped(): + s3_outage("stop", "start", "stopped (connection refused / name not resolved)") + + +def test_s3_frozen(): + s3_outage("pause", "unpause", "frozen (no answer, timeouts)") + + +def test_crash_between_s3_and_db(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"])) + check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}") + att_id, key = result["attachment_id"], result["object_key"] + state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'") + check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3") + print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3") + call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"]) + post_message(alice, chat["id"], "x", [att_id], expected=404) + print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404") + + report = janitor() + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key), + "fresh upload was deleted inside the grace period") + print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> " + "fresh row and object are kept (the upload might still be running)") + backdate(f"id = '{att_id}'") + report = janitor("--dry-run") + check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row") + print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted") + report = janitor() + check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists") + check(not object_exists(key), "object still exists") + print(f"janitor: {json.dumps(report)} -> row and object deleted") + report = janitor() + print(f"janitor again: {json.dumps(report)} -> idempotent") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment damaged by cleanup") + print("the ready attachment of the same chat is untouched") + print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period") + + +def test_orphan_objects(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + key = last_json(app_python(PUT_ORPHAN))["key"] + print(f"object {key} put into the bucket without a row in PostgreSQL") + # Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с. + short = {"UPLOAD_TIMEOUT_SECONDS": "1"} + report = janitor("--grace-seconds", "3", env=short) + check(object_exists(key), "fresh orphan deleted inside grace period") + print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept") + time.sleep(4) + report = janitor("--grace-seconds", "3", "--dry-run", env=short) + check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key), + "dry run must report and keep the orphan") + print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept") + report = janitor("--grace-seconds", "3", env=short) + check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted") + print(f"janitor: {json.dumps(report)} -> orphan deleted") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment was deleted with orphans") + print("ready attachments older than the grace period are untouched (they have rows)") + + victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json + victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'") + app_python(DELETE_OBJECT, victim_key) + report = janitor("--grace-seconds", "3", env=short) + check(report["missing_objects"] >= 1, "missing object not reported") + check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted") + reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"]) + print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects=" + f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)") + print("PASS: orphans are found and deleted only after the grace period; missing objects are reported") + + +def api_tmp_files(): + counts = {} + for container in compose_output("ps", "-q", "api").split(): + counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0) + return counts + + +def test_aborted_upload_leaves_nothing(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + objects = last_json(app_python(COUNT_OBJECTS))["objects"] + rows = int(sql_value("SELECT count(*) FROM attachments")) + for sent in (100_000, 3_000_000): + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n" + f"Content-Length: 9000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(sent)) + sock.close() + print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection") + time.sleep(2) + check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row") + check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object") + print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}") + tmp = api_tmp_files() + check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}") + print(f"temporary upload files in api containers: {tmp} (removed when the request ends)") + for container in compose_output("ps", "-q", "api").split(): + mounts = docker("inspect", "-f", "{{json .Mounts}}", container) + # файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных + storage = [m for m in json.loads(mounts) + if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)] + check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}") + print("api containers have no volumes: the original is never stored locally") + print("PASS: an interrupted upload leaves no row, no object and no local file") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db], + "orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]} + groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"] + check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]") + wait_ready() + run_tests(groups[target]) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 0000000..3d71fcc --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,101 @@ +import time +import uuid + +from common import call, check, compose_output, new_user, psql, run_tests + + +def sql_value(sql): + code, output, error = psql(sql) + check(code == 0, f"psql failed: {error}") + return output + + +def test_create_chat_rolls_back(): + alice = new_user("rollback") + title = "rollback-" + uuid.uuid4().hex[:8] + inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + call("POST", "/api/v1/chats", 400, token=alice["token"], + body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) + + left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") + orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") + inserted_after = inserted_before + deadline = time.monotonic() + 15 + while inserted_after == inserted_before and time.monotonic() < deadline: + time.sleep(0.5) + inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; " + f"chats insert counter: {inserted_before} -> {inserted_after}") + check(left == "0" and orphans == "0", "chat row survived the failed transaction") + check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown") + print("PASS: chat row was inserted and then rolled back together with members") + + +def test_message_needs_membership_in_db(): + code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") + print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") + check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") + print("PASS: the database itself rejects a message from a non-member") + + +def test_password_is_argon2id(): + user = new_user("hash") + stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'") + print(f"stored password: {stored[:31]}... (length {len(stored)})") + check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash") + print("PASS: password is stored as Argon2id hash") + + +def test_unique_username_constraint(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'users'::regclass AND contype = 'u'") + print(f"unique constraint on users: {constraint}") + check(constraint == "UNIQUE (username)", "no unique constraint on username") + print("PASS: username is unique in the database") + + +def test_migrations_are_not_repeated(): + versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + print(f"applied migrations: {versions}; running the migrate service once more") + output = compose_output("run", "--rm", "--no-deps", "-T", "migrate") + again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1] + print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}") + check(again == versions and "none" in last_log, "migrations were applied again") + print("PASS: migrations are applied once, a repeated run does nothing") + + +def test_replicas_do_not_migrate(): + """lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему.""" + logs = compose_output("logs", "api").splitlines() + checked = [line for line in logs if "Schema is up to date" in line] + migrated = [line for line in logs if "Migrations applied" in line] + migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': " + f"{len(migrated)}; migrate service: {migrate_exit}") + check(checked and not migrated, "api replicas must not apply migrations") + check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0") + print("PASS: only the migrate service applies migrations, replicas wait for the schema") + + +def test_idempotency_key_is_unique_in_db(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conname = 'messages_idempotency_key'") + print(f"unique constraint for Idempotency-Key: {constraint}") + check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys") + print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database") + + +if __name__ == "__main__": + run_tests([ + test_create_chat_rolls_back, + test_message_needs_membership_in_db, + test_password_is_argon2id, + test_unique_username_constraint, + test_migrations_are_not_repeated, + test_replicas_do_not_migrate, + test_idempotency_key_is_unique_in_db, + ]) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..e6017ce --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,57 @@ +import datetime as dt +import os +import time + +from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready + +SHORT_TTL = 2 + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env + else: + env["SESSION_TTL_SECONDS"] = str(ttl) + compose("up", "-d", "--no-deps", "--wait", "api", env=env) + wait_ready() + + +def seconds_left(expires_at): + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def test_session_expires(): + print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}") + restart_api(SHORT_TTL) + try: + user = new_user("ttl") + session = login(user["username"], user["password"]).json + token = session["token"] + left = seconds_left(session["expires_at"]) + print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: " + f"{redis_cli('pttl', session_key(token))} ms") + check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions") + + call("GET", "/api/v1/users/me", 200, token=token) + print("before expiry: GET /users/me -> 200") + + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0) + call("GET", "/api/v1/users/me", 401, token=token) + exists = redis_cli("exists", session_key(token)) + print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}") + check(exists == "0", "Redis must delete the expired session key") + finally: + print("recreating api with SESSION_TTL_SECONDS from .env") + restart_api() + + user = new_user("ttl_back") + left = seconds_left(login(user["username"], user["password"]).json["expires_at"]) + print(f"normal API again: new session expires in {left:.0f} s") + check(left > 300, "normal TTL was not restored") + print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401") + + +if __name__ == "__main__": + run_tests([test_session_expires]) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 0000000..8eaef38 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,110 @@ +import sys +import time +import uuid + +from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready + +FAKE_TOKEN = "A" * 43 + + +def timed(method, path, expected, **kwargs): + started = time.monotonic() + reply = call(method, path, expected, **kwargs) + return reply, (time.monotonic() - started) * 1000 + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + before = send(alice, chat["id"], "confirmed before outage").json + return alice, bob, chat, before + + +def recovered(alice, chat, before): + wait_ready() + history, _ = read_history(alice, chat["id"]) + check(before in history, "message confirmed before the outage is lost or changed") + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the old token works, confirmed messages are in place") + + +def test_redis_down(): + alice, bob, chat, before = prepare() + print("stopping redis") + compose("stop", "redis") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") + call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) + print("unknown token -> 503, not 200: nobody is trusted without a check") + call("GET", "/api/v1/chats", 401) + print("no token -> 401") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + print("login -> 503: there is nowhere to store the session") + call("POST", "/api/v1/users", 201, + body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("registration -> 201: it needs only PostgreSQL") + finally: + print("starting redis") + compose("start", "redis") + recovered(alice, chat, before) + print("PASS: Redis outage gives 503 and does not let anyone in without a session check") + + +def test_redis_crash(): + alice = new_user("crash") + print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") + compose("kill", "redis") + try: + # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. + status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status + print(f"right after the crash: GET /users/me -> {status}") + finally: + print("starting redis") + compose("start", "redis") + wait_ready() + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the session issued before the crash still works (AOF, appendfsync always)") + print("PASS: sessions survive a Redis crash, not only a clean restart") + + +def test_postgres_down(): + alice, bob, chat, before = prepare() + print("stopping postgres") + compose("stop", "postgres") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + for method, path, body in (("GET", "/api/v1/users/me", None), + ("GET", "/api/v1/chats", None), + ("GET", f"/api/v1/chats/{chat['id']}/messages", None), + ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): + reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + check("postgres" not in reply.data.decode().lower(), "error must not show database details") + print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") + call("GET", "/api/v1/chats", 401) + call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) + print("no token -> 401, unknown token -> 401: anonymous users are not let in") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + call("POST", "/api/v1/users", 503, + body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("login -> 503, registration -> 503") + finally: + print("starting postgres") + compose("start", "postgres") + recovered(alice, chat, before) + print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], + "all": [test_redis_down, test_redis_crash, test_postgres_down]} + check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") + run_tests(tests[target]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..407dea3 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,174 @@ +import json +import socket +import threading +import time +import urllib.parse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send + +PARALLEL = 20 + + +def post_raw(user, chat_id, raw, key): + """POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"], + raw=raw, headers={"Idempotency-Key": key}) + + +def texts_in(user, chat_id): + history, _ = read_history(user, chat_id) + return [message["text"] for message in history] + + +def test_replay_returns_same_message(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "Привет 👋", key=key) + # То же JSON-значение, записанное иначе: пробелы и \u-экранирование. + same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key) + again = send(alice, chat["id"], "Привет 👋", key=key) + print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}") + check(same.status == 201 and same.json == first.json and again.json == first.json, + "retry must return the original 201 and the same message") + check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate") + print("PASS: a retry with the same key returns the original 201 and message, no duplicate") + + +def test_same_key_parallel_through_replicas(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(_): + barrier.wait() # все запросы стартуют одновременно + return send(alice, chat["id"], "one message for 20 requests", key=key) + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + replies = list(pool.map(post, range(PARALLEL))) + bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies} + served = Counter(reply.instance for reply in replies) + count = texts_in(bob, chat["id"]).count("one message for 20 requests") + print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, " + f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}") + check(len(served) >= 2, "requests did not go through different replicas") + check(len(bodies) == 1 and count == 1, "parallel retries created more than one message") + print("PASS: parallel requests with one key through different replicas create exactly one message") + + +def test_same_key_different_bodies_parallel(): + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(index): + text = "body A" if index % 2 == 0 else "body B" + barrier.wait() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"], + body={"text": text}, headers={"Idempotency-Key": key}) + return text, reply + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(post, range(PARALLEL))) + texts = texts_in(alice, chat["id"]) + winner = [t for t in texts if t in ("body A", "body B")] + print(f"{PARALLEL} parallel POSTs, one key, two different bodies: " + f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}") + check(len(winner) == 1, "exactly one message must be created") + for text, reply in results: + expected = 201 if text == winner[0] else 409 + check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}") + print("PASS: the first body wins, the same body gets 201, the other body gets 409") + + +def test_key_scope(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + other = new_chat(alice, bob) + key = new_key() + a1 = send(alice, chat["id"], "scope", key=key).json + b1 = send(bob, chat["id"], "scope", key=key).json + a2 = send(alice, other["id"], "scope", key=key).json + print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages") + check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat") + call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"], + body={"text": "other text"}, headers={"Idempotency-Key": key}) + print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409") + + +def test_key_validation(): + alice = new_user("alice") + chat = new_chat(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": "no key"}) + for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"): + call("POST", path, 400, token=alice["token"], body={"text": "bad key"}, + headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8 + longest = ("Az09._:-" * 16)[:128] + call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest}) + print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201") + print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}") + + +def test_failed_request_does_not_burn_key(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key}) + call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key}) + created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key}) + print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}") + print("PASS: rejected requests (400, 404) do not store a result for the key") + + +def test_replay_checks_session_and_access(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + send(alice, chat["id"], "private", key=key) + stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}") + print("PASS: a retry checks the session and chat access again, the saved answer is not leaked") + + +def test_lost_response_then_retry(): + """Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен.""" + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + body = json.dumps({"text": "answer was lost"}).encode("utf-8") + url = urllib.parse.urlsplit(BASE_URL) + request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection: + connection.sendall(request + body) # и сразу закрываем, ответ не читаем + time.sleep(1) + retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"], + body={"text": "answer was lost"}, headers={"Idempotency-Key": key}) + count = texts_in(alice, chat["id"]).count("answer was lost") + print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}") + check(count == 1, "the retry after a lost answer created a duplicate") + print("PASS: an unconfirmed POST can be safely retried with the same key") + + +if __name__ == "__main__": + run_tests([ + test_replay_returns_same_message, + test_same_key_parallel_through_replicas, + test_same_key_different_bodies_parallel, + test_key_scope, + test_key_validation, + test_failed_request_does_not_burn_key, + test_replay_checks_session_and_access, + test_lost_response_then_retry, + ]) diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 0000000..4af44fa --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,57 @@ +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send + +CLIENTS = 10 +MESSAGES_PER_CLIENT = 10 +REGISTRATIONS = 20 + + +def test_concurrent_messages(): + users = [new_user(f"race{i}") for i in range(CLIENTS)] + chat = new_chat(*users, title="race") + barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно + + def client(index): + barrier.wait() + for n in range(MESSAGES_PER_CLIENT): + send(users[index], chat["id"], f"client-{index}-message-{n}") + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков + elapsed = time.monotonic() - started + + history, pages = read_history(users[0], chat["id"], limit=30) + ids = {message["id"] for message in history} + expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)} + print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; " + f"read {len(history)} in {pages} pages, unique ids: {len(ids)}") + check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages") + check(len(ids) == len(history), "duplicate message ids") + check({message["text"] for message in history} == expected, "lost or changed message texts") + print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates") + + +def test_concurrent_registration(): + username = "race_" + uuid.uuid4().hex[:10] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + barrier.wait() + body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD} + return call("POST", "/api/v1/users", (201, 409), body=body).status + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + statuses = list(pool.map(register, range(REGISTRATIONS))) + created, conflicts = statuses.count(201), statuses.count(409) + print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}") + check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win") + print("PASS: concurrent registration created exactly one user") + + +if __name__ == "__main__": + run_tests([test_concurrent_messages, test_concurrent_registration]) diff --git a/student-tests/test_s3_security.py b/student-tests/test_s3_security.py new file mode 100644 index 0000000..ece71a3 --- /dev/null +++ b/student-tests/test_s3_security.py @@ -0,0 +1,146 @@ +"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен. + +Нужен запущенный стенд из compose.yaml. +""" +import json +import socket + +import images +from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload + +# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API. +ANONYMOUS = r''' +import json, sys, urllib.request, urllib.error +import settings +base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET +def status(method, url, data=None): + try: + with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r: + return r.status, r.read(300).decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read(300).decode("utf-8", "replace") +out = {} +for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None), + ("head_object", "HEAD", base + "/" + sys.argv[1], None), + ("list_bucket", "GET", base + "?list-type=2", None), + ("put_object", "PUT", base + "/originals/anonymous-test", b"x"), + ("get_bucket_policy", "GET", base + "?policy", None)): + code, body = status(method, url, data) + out[name] = {"status": code, "s3_code": body.split("")[1].split("")[0] if "" in body else ""} +print(json.dumps(out)) +''' + +# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json). +API_KEY_RIGHTS = r''' +import io, json, sys, uuid +from botocore.exceptions import ClientError +import settings, storage +c, bucket = storage._client, settings.S3_BUCKET +def attempt(name, fn): + try: + fn(); return "allowed" + except ClientError as e: + return "denied: " + e.response.get("Error", {}).get("Code", "?") +public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) +key = "originals/" + uuid.uuid4().hex +result = { + "get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()), + "put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")), + "delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)), + "list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)), + "create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])), + "make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)), + "read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)), + "delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)), + "change versioning": attempt("versioning", lambda: c.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})), +} +try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket + result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", [])) +except ClientError as e: + result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?") +print(json.dumps(result)) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + att = upload(alice, chat["id"], images.sample_png()).json + key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + return alice, bob, att, key + + +def test_bucket_is_private(): + alice, bob, att, key = prepare() + result = last_json(app_python(ANONYMOUS, key)) + for name, reply in result.items(): + print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}") + check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}") + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + print("the same file through the API for a chat member -> 200") + print("PASS: the bucket is private: no anonymous read, list, write or policy read") + + +def test_api_key_is_limited(): + alice, bob, att, key = prepare() + result = last_json(app_python(API_KEY_RIGHTS, key)) + allowed = {"get own object", "put object", "delete object", "list bucket"} + visible = result.pop("visible buckets") + print(f"API key: {'list buckets':<32} -> {visible}") + check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets") + for name, outcome in result.items(): + print(f"API key: {name:<32} -> {outcome}") + if name in allowed: + check(outcome == "allowed", f"API key must be allowed to {name}") + else: + check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}") + print("PASS: the API key can only get/put/delete/list objects of its own bucket; " + "bucket admin, policy and other buckets are denied") + + +def test_no_open_ports_or_console(): + config = json.loads(compose_output("config", "--format", "json")) + s3 = config["services"]["s3"] + check(not s3.get("ports"), "s3 must not publish ports") + check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, " + "filer UI 8888 are reachable only inside the stand)") + for port in (8333, 9333, 8888, 19333): + sock = socket.socket() + sock.settimeout(1) + refused = sock.connect_ex(("127.0.0.1", port)) != 0 + sock.close() + check(refused, f"127.0.0.1:{port} is reachable from the host") + print(f"127.0.0.1:{port} from the host -> connection refused") + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()} + for name, items in sorted(secrets.items()): + print(f"secrets of {name}: {items or '-'}") + for name in ("api", "janitor"): + check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"}, + f"{name} must not get S3 admin keys or the key config") + check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key") + print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init") + + +def test_bootstrap_is_idempotent(): + alice, bob, att, key = prepare() + output = compose_output("run", "--rm", "--no-deps", "s3-init") + print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip())) + check("bucket is private" in output, "bootstrap did not confirm a private bucket") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "file changed after repeated bootstrap") + upload(alice, att["chat_id"], images.sample_png()) + print("after the repeated bootstrap: old file downloads, new upload works") + print("PASS: bucket/user/policy bootstrap can be repeated safely") + + +if __name__ == "__main__": + run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console, + test_bootstrap_is_idempotent]) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..42dcb5e --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,218 @@ +"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик. + +Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что +тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково +(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше +DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование) +при тех же 10 загрузках не помещается в лимит 512 MiB. + +Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков +одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status +(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера +(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с. + + python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json] +""" +import argparse +import hashlib +import json +import platform +import statistics +import threading +import time + +import images +from common import ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_balanced, wait_ready + + +def proc_status(container): + """VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты).""" + script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; " + "for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current " + "/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; " + "do [ -r $f ] && echo \"$f $(cat $f)\"; done; true") + out = docker("exec", container, "sh", "-c", script) + values = {} + for line in out.splitlines(): + parts = line.replace(":", " ").split() + if parts[0] in ("VmRSS", "VmHWM"): + values[parts[0]] = int(parts[1]) + elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")): + values["cgroup_peak"] = int(parts[1]) + elif parts[0].endswith(("memory.current", "usage_in_bytes")): + values["cgroup_current"] = int(parts[1]) + return values + + +class Sampler(threading.Thread): + def __init__(self, containers): + super().__init__(daemon=True) + self.containers = containers + self.stop = threading.Event() + self.rss = {c: 0 for c in containers} + self.cgroup = {c: 0 for c in containers} + self.samples = 0 + + def run(self): + while not self.stop.is_set(): + for c in self.containers: + try: + v = proc_status(c) + except Exception: # noqa: BLE001 — один пропущенный замер не важен + continue + self.rss[c] = max(self.rss[c], v.get("VmRSS", 0)) + self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0)) + self.samples += 1 + + +def api_containers(): + from common import api_containers as running + return running() + + +def fresh_replicas(): + compose("restart", "api") + containers = wait_balanced(2, seconds=90) + wait_ready() + return containers + + +def warm_up(user, chat): + """По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п.""" + for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4: + upload(user, chat["id"], data, ctype) + + +def scenario(name, payloads, concurrency): + user = new_user("mem") + chat = new_chat(user) + containers = fresh_replicas() + warm_up(user, chat) + before = {c: proc_status(c) for c in containers} + sampler = Sampler(containers) + sampler.start() + barrier = threading.Barrier(concurrency) + results = [None] * concurrency + + def worker(i): + data, ctype = payloads[i % len(payloads)] + barrier.wait() + started = time.monotonic() + try: + reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180) + results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data) + except AssertionError as exc: + results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)] + wall = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - wall + time.sleep(0.5) + sampler.stop.set() + sampler.join() + after = {c: proc_status(c) for c in containers} + + ok = [r for r in results if r[0] == 201] + check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 201]}") + for status, _, _, att, data in ok[:3]: + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"]) + check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs") + latencies = sorted(r[1] for r in results) + per_replica = [] + for c in containers: + b, a = before[c], after[c] + per_replica.append({ + "replica": c[:12], + "uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])), + "rss_before_mib": round(b["VmRSS"] / 1024, 1), + "rss_peak_mib": round(a["VmHWM"] / 1024, 1), + "rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1), + "rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1), + "cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None, + "cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None, + }) + sizes = sorted({len(d) for d, _ in payloads}) + summary = { + "scenario": name, + "concurrency": concurrency, + "file_bytes": sizes, + "total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1), + "all_201": True, + "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)}, + "sampler_rounds": sampler.samples, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False)) + return summary + + +# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование. +NAIVE_VS_STREAMING = r''' +import json, os, subprocess, sys +code = r""" +import io, os, resource, struct, sys, zlib +sys.path.insert(0, os.getcwd()) +os.environ.setdefault("S3_ACCESS_KEY_ID", "x") +from PIL import Image +import uploads # импорт до замера: сравниваем только саму проверку файла +Image.MAX_IMAGE_PIXELS = None +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +w, h = 8000, 5000 +comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3) +idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush() +data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") +del idat, comp, row +base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +if sys.argv[1] == "naive": + body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB) + im = Image.open(io.BytesIO(data)); im.load() +else: + f = io.BytesIO(data); uploads.check_png(f) +print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024) +""" +out = {} +for mode in ("naive", "streaming"): + r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd()) + out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:] +print(json.dumps(out)) +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default="evidence/upload-memory.json") + args = parser.parse_args() + check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads") + + print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}") + print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...") + large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)] + pixels = [(images.png(8000, 5000), "image/png")] + jpeg = [(images.sample_jpeg(), "image/jpeg")] + print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes") + + runs = [ + scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency), + scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency), + scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency), + ] + naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1]) + print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) " + f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB") + limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split() + result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20, + "api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive} + (ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"]) + print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB " + f"(limit {result['api_limit_mib']} MiB); results in {args.output}") + + +if __name__ == "__main__": + main() diff --git a/uploads.py b/uploads.py new file mode 100644 index 0000000..2452212 --- /dev/null +++ b/uploads.py @@ -0,0 +1,351 @@ +"""Приём и проверка загружаемого изображения с ограниченной памятью. + +1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше + MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном + файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса. +2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях + читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково + (CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма), + JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение. +""" +import asyncio +import hashlib +import logging +import struct +import tempfile +import threading +import time +import unicodedata +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image +from starlette.requests import ClientDisconnect, Request + +import settings +from errors import bad_request, payload_too_large, unsupported_media_type + +log = logging.getLogger("messenger.uploads") + +PNG = "image/png" +JPEG = "image/jpeg" +ALLOWED_TYPES = (PNG, JPEG) +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" + +SPOOL_IN_MEMORY = 256 * 1024 +INFLATE_CHUNK = 64 * 1024 +CRC_CHUNK = 64 * 1024 + +# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке). +Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS + +# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY. +_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY) + + +@dataclass +class Upload: + file: BinaryIO + size: int + sha256: str + + +@dataclass +class ImageInfo: + content_type: str + width: int + height: int + + +class Damaged(Exception): + pass + + +class TooManyPixels(Exception): + pass + + +# ---------- имя файла и заголовки ---------- + +def validate_filename(value: str | None) -> str: + """filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов.""" + if value is None: + raise bad_request("Query parameter filename is required") + if not 1 <= len(value) <= 255 or not value.strip(): + raise bad_request("filename must be 1-255 characters") + if value in (".", ".."): + raise bad_request("filename is not allowed") + for char in value: + if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"): + raise bad_request("filename contains forbidden characters") + return value + + +def declared_type(header: str | None) -> str: + base = (header or "").split(";", 1)[0].strip().lower() + if base not in ALLOWED_TYPES: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return base + + +def declared_length(header: str | None) -> int | None: + if header is None: + return None + if not header.isdigit(): + raise bad_request("Invalid Content-Length") + length = int(header) + if length > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + return length + + +# ---------- приём тела ---------- + +async def receive(request: Request) -> Upload: + """Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется.""" + spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR) + digest = hashlib.sha256() + size = 0 + deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS + stream = request.stream().__aiter__() + try: + while True: + wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic()) + if wait <= 0: + raise bad_request("Upload took too long") + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except StopAsyncIteration: + break + except asyncio.TimeoutError: + raise bad_request("Upload stalled or took too long") from None + if not chunk: + continue + size += len(chunk) + if size > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + digest.update(chunk) + spool.write(chunk) + except ClientDisconnect: + spool.close() + log.info("Upload aborted by the client after %s bytes, nothing stored", size) + raise bad_request("Upload was interrupted") from None + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise bad_request("Empty body") + return Upload(spool, size, digest.hexdigest()) + + +async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None: + """Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв. + + Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается. + """ + read = 0 + deadline = time.monotonic() + seconds + try: + stream = request.stream().__aiter__() + while read <= limit: + wait = deadline - time.monotonic() + if wait <= 0: + return + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except (StopAsyncIteration, asyncio.TimeoutError): + return + read += len(chunk) + except (ClientDisconnect, RuntimeError): + return + + +# ---------- проверка содержимого ---------- + +def sniff(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return PNG + if head.startswith(JPEG_SIGNATURE): + return JPEG + return None + + +def inspect(upload: Upload, declared: str) -> ImageInfo: + upload.file.seek(0) + detected = sniff(upload.file.read(16)) + if detected is None: + raise unsupported_media_type("File is not a PNG or JPEG image") + if detected != declared: + raise unsupported_media_type("Content-Type does not match the file content") + upload.file.seek(0) + try: + with _decode_slots: + width, height = check_png(upload.file) if detected == PNG else check_jpeg(upload.file) + except TooManyPixels: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except Damaged as exc: + log.info("Rejected damaged %s: %s", detected, exc) + raise unsupported_media_type("Image is damaged or not supported") from None + except Image.DecompressionBombError: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc: + log.info("Rejected damaged %s: %s", detected, type(exc).__name__) + raise unsupported_media_type("Image is damaged or not supported") from None + finally: + upload.file.seek(0) + return ImageInfo(detected, width, height) + + +def check_pixels(width: int, height: int) -> None: + if width < 1 or height < 1: + raise Damaged("zero dimension") + if width * height > settings.MAX_IMAGE_PIXELS: + raise TooManyPixels() + + +def check_jpeg(file: BinaryIO) -> tuple[int, int]: + with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS + width, height = image.size + check_pixels(width, height) + # Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8: + # 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB. + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + return width, height + + +# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR). +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +FILTER_BYTES = bytes(range(5)) + + +class PngRows: + """Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки.""" + + def __init__(self, width, height, depth, color, interlace): + bits = depth * PNG_CHANNELS[color] + + def row_bytes(w): + return (w * bits + 7) // 8 + + passes = [(width, height)] if interlace == 0 else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + # (длина строки с байтом фильтра, байт в проходе) + self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0] + self.expected = sum(total for _, total in self.passes) + self.index, self.pos = 0, 0 + self.total = 0 + + def feed(self, data: bytes) -> None: + view = memoryview(data) + self.total += len(data) + if self.total > self.expected: + raise Damaged("more pixel data than the header says") + offset = 0 + while offset < len(view): + stride, size = self.passes[self.index] + take = min(len(view) - offset, size - self.pos) + first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride + if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES): + raise Damaged("invalid filter type") + self.pos += take + offset += take + if self.pos == size: + self.index, self.pos = self.index + 1, 0 + + def done(self) -> bool: + return self.total == self.expected + + +def check_png(file: BinaryIO) -> tuple[int, int]: + if file.read(8) != PNG_SIGNATURE: + raise Damaged("signature") + rows = inflater = None + width = height = 0 + seen_idat = idat_finished = have_plte = False + color = None + first = True + while True: + header = file.read(8) + if len(header) < 8: + raise Damaged("truncated: no IEND") + length, kind = struct.unpack(">I4s", header) + if length > 0x7FFFFFFF: + raise Damaged("chunk length") + if first and kind != b"IHDR": + raise Damaged("IHDR must be first") + crc = zlib.crc32(kind) + keep = kind in (b"IHDR",) + kept = b"" + left = length + while left: + piece = file.read(min(CRC_CHUNK, left)) + if not piece: + raise Damaged("truncated chunk") + left -= len(piece) + crc = zlib.crc32(piece, crc) + if keep: + kept += piece + elif kind == b"IDAT": + if rows is None or idat_finished: + raise Damaged("IDAT out of order") + feed_inflate(inflater, rows, piece) + stored = file.read(4) + if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: + raise Damaged(f"CRC mismatch in {kind!r}") + + if kind == b"IHDR": + if not first or length != 13: + raise Damaged("IHDR") + width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) + if width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise Damaged("dimensions") + check_pixels(width, height) # до распаковки + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]: + raise Damaged("color type / bit depth") + if compression != 0 or filter_method != 0 or interlace not in (0, 1): + raise Damaged("IHDR methods") + rows = PngRows(width, height, depth, color, interlace) + inflater = zlib.decompressobj() + elif kind == b"PLTE": + have_plte = True + elif kind == b"IDAT": + seen_idat = True + elif kind == b"IEND": + break + if seen_idat and kind != b"IDAT": + idat_finished = True + first = False + + if file.read(1): + raise Damaged("data after IEND") + if color == 3 and not have_plte: + raise Damaged("palette image without PLTE") + if not seen_idat: + raise Damaged("no IDAT") + tail = inflater.flush(INFLATE_CHUNK) + if tail: + rows.feed(tail) + if not inflater.eof or inflater.unused_data or not rows.done(): + raise Damaged("pixel data is incomplete") + return width, height + + +def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: + """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" + if inflater.eof: + if data: + raise Damaged("data after the end of zlib stream") + return + out = inflater.decompress(data, INFLATE_CHUNK) + rows.feed(out) + while inflater.unconsumed_tail and not inflater.eof: + out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK) + rows.feed(out) + if inflater.eof and inflater.unused_data: + raise Damaged("data after the end of zlib stream")