From 5c3a31db2edf93920bfc909faf190d05c91f28bb Mon Sep 17 00:00:00 2001 From: kalibr456 <145665723+kalibr456@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:25:10 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=9B=D0=A0=204:=20S3-=D1=85=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B5=20(SeaweedFS)=20=D0=B8=20?= =?UTF-8?q?=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 7 + .env.example | 27 +- .gitattributes | 6 + .github/pull_request_template.md | 16 +- .gitignore | 1 + Dockerfile | 27 ++ REPORT.md | 228 +++++++++- attachments.py | 72 +++ auth.py | 139 ++++++ compose.yaml | 211 +++++++++ db.py | 359 +++++++++++++++ errors.py | 40 ++ evidence/attachments.txt | 67 +++ evidence/balancing.txt | 18 + evidence/check.txt | 6 + evidence/consistency.txt | 47 ++ evidence/db.txt | 21 + evidence/explain-history.txt | 99 +++++ evidence/explain.txt | 43 ++ evidence/failover.txt | 33 ++ evidence/failures.txt | 29 ++ evidence/idempotency.txt | 22 + evidence/load.json | 17 + evidence/load.txt | 22 + evidence/persistence.txt | 24 + evidence/race.txt | 10 + evidence/redis.txt | 14 + evidence/s3-security.txt | 44 ++ evidence/smoke.txt | 6 + evidence/stack.txt | 40 ++ evidence/student-tests.txt | 26 ++ evidence/summary.md | 40 ++ evidence/upload-memory.json | 124 ++++++ evidence/upload-memory.txt | 13 + infra/haproxy.cfg | 77 ++++ infra/redis.conf | 13 + infra/s3-init.sh | 57 +++ janitor.py | 162 +++++++ main.py | 583 +++++++++++++++++++++++++ migrate.py | 8 + migrations/001_init.sql | 33 ++ migrations/002_text_as_bytes.sql | 11 + migrations/003_idempotency.sql | 11 + migrations/004_attachments.sql | 48 ++ requirements.txt | 29 ++ s3_init.py | 103 +++++ scripts/init_secrets.py | 77 ++++ scripts/restart.cmd | 13 + scripts/restart.sh | 13 + scripts/stop-one.cmd | 6 + scripts/stop-one.sh | 5 + scripts/stop_one.py | 29 ++ settings.py | 86 ++++ storage.py | 153 +++++++ student-tests/check_stack.py | 78 ++++ student-tests/collect_evidence.py | 293 +++++++++++++ student-tests/common.py | 324 ++++++++++++++ student-tests/demo_instances.py | 16 + student-tests/explain.py | 28 ++ student-tests/explain.sql | 76 ++++ student-tests/failover_metrics.py | 129 ++++++ student-tests/fixtures/progressive.jpg | Bin 0 -> 1081 bytes student-tests/fixtures/sample.jpg | Bin 0 -> 1492 bytes student-tests/images.py | 74 ++++ student-tests/loadgen.py | 102 +++++ student-tests/redis_info.py | 29 ++ student-tests/rolling_update.py | 88 ++++ student-tests/run_failover.py | 55 +++ student-tests/run_persistence.py | 47 ++ student-tests/test_api.py | 258 +++++++++++ student-tests/test_attachments.py | 268 ++++++++++++ student-tests/test_balancing.py | 103 +++++ student-tests/test_consistency.py | 275 ++++++++++++ student-tests/test_db.py | 101 +++++ student-tests/test_expiry.py | 57 +++ student-tests/test_failures.py | 110 +++++ student-tests/test_idempotency.py | 174 ++++++++ student-tests/test_race.py | 57 +++ student-tests/test_s3_security.py | 146 +++++++ student-tests/upload_memory.py | 218 +++++++++ uploads.py | 351 +++++++++++++++ 81 files changed, 6847 insertions(+), 25 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 attachments.py create mode 100644 auth.py create mode 100644 compose.yaml create mode 100644 db.py create mode 100644 errors.py create mode 100644 evidence/attachments.txt create mode 100644 evidence/balancing.txt create mode 100644 evidence/check.txt create mode 100644 evidence/consistency.txt create mode 100644 evidence/db.txt create mode 100644 evidence/explain-history.txt create mode 100644 evidence/explain.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/failures.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/load.json create mode 100644 evidence/load.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/race.txt create mode 100644 evidence/redis.txt create mode 100644 evidence/s3-security.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stack.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/summary.md create mode 100644 evidence/upload-memory.json create mode 100644 evidence/upload-memory.txt create mode 100644 infra/haproxy.cfg create mode 100644 infra/redis.conf create mode 100644 infra/s3-init.sh create mode 100644 janitor.py create mode 100644 main.py create mode 100644 migrate.py create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_text_as_bytes.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 requirements.txt create mode 100644 s3_init.py create mode 100644 scripts/init_secrets.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 settings.py create mode 100644 storage.py create mode 100644 student-tests/check_stack.py create mode 100644 student-tests/collect_evidence.py create mode 100644 student-tests/common.py create mode 100644 student-tests/demo_instances.py create mode 100644 student-tests/explain.py create mode 100644 student-tests/explain.sql create mode 100644 student-tests/failover_metrics.py create mode 100644 student-tests/fixtures/progressive.jpg create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/images.py create mode 100644 student-tests/loadgen.py create mode 100644 student-tests/redis_info.py create mode 100644 student-tests/rolling_update.py create mode 100644 student-tests/run_failover.py create mode 100644 student-tests/run_persistence.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_balancing.py create mode 100644 student-tests/test_consistency.py create mode 100644 student-tests/test_db.py create mode 100644 student-tests/test_expiry.py create mode 100644 student-tests/test_failures.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_race.py create mode 100644 student-tests/test_s3_security.py create mode 100644 student-tests/upload_memory.py create mode 100644 uploads.py diff --git a/.dockerignore b/.dockerignore index bc16e556..82449c19 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,10 @@ __pycache__ node_modules evidence .DS_Store +tests +student-tests +contracts +scripts +infra +examples +*.md diff --git a/.env.example b/.env.example index d99d26a2..9233482e 100644 --- a/.env.example +++ b/.env.example @@ -7,14 +7,37 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 + +# Pools and timeouts (same as lab3) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_WAIT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_POOL_MAX=20 +REDIS_TIMEOUT_MS=500 +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=10 + +# lab4: S3 client and uploads +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=5 +S3_MAX_ATTEMPTS=2 +S3_POOL_MAX=20 +UPLOAD_TIMEOUT_SECONDS=60 +UPLOAD_IDLE_TIMEOUT_SECONDS=10 +DECODE_CONCURRENCY=2 +# Cleanup of abandoned uploads (janitor service): grace period and pass interval +ATTACHMENT_GRACE_SECONDS=3600 +JANITOR_INTERVAL_SECONDS=300 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..3fd80096 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.example text eol=lf +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..7f842876 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,14 +1,14 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Бабиченко Максим, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесён весь lab3 (API на FastAPI, PostgreSQL, сессии в Redis, HAProxy + 2 реплики, идемпотентность, migrate, свои тесты). Добавлены: SeaweedFS (`s3`) с приватным bucket и volume `s3data` (MinIO недоступен в публичных реестрах); идемпотентный bootstrap `s3-init`; отдельный ключ API с правами только на свой bucket; миграция 004 (`attachments`, `message_attachments` с составными FK по `chat_id`); синхронная загрузка PNG/JPEG с потоковыми лимитами (10 MiB, 40 Мпикс до распаковки, MIME по содержимому, проверка целостности без декодирования в память), скачивание через API с проверкой участника; `attachment_ids` в сообщениях; запись «строка-намерение -> PUT -> ready» и сервис `janitor` (очистка брошенных загрузок и объектов-сирот с grace period, dry-run, отчёт о missing). Подробности — REPORT.md. +Команды запуска из чистого clone: `Copy-Item .env.example .env; python scripts/init_secrets.py; docker compose up -d --build --wait; python student-tests/collect_evidence.py` (полностью — REPORT.md, раздел «Запуск из чистого clone»). +Результаты проверок и ссылка на REPORT.md: evidence/summary.md и evidence/*.txt; таблицы замеров — REPORT.md, раздел «Эксперименты и отказы». +Ограничения / известные ошибки: один узел (lb, postgres, redis, s3 — SPOF); повтор загрузки после потерянного ответа создаёт второе вложение (так по контракту); missing-объекты janitor только находит. См. REPORT.md, «Ограничения». +Источники и использование ИИ: библиотеки и документация перечислены в REPORT.md; при написании кода и тестов использовался ИИ-ассистент (Claude), код проверен и прогнан мной. - [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [ ] Контракт и публичные тесты не ослаблены. diff --git a/.gitignore b/.gitignore index b5a71896..0a6ee0a6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/ backups/ # Runtime/build artifacts .venv/ +venv/ __pycache__/ *.py[cod] node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b96e820a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt + +COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py ./ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser + +USER appuser + +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)" + +CMD ["python", "main.py"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..527b32b8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,235 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бабиченко Максим, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран +- Commit/PR: <<ссылка на MR в Gitea после публикации>> +- Использованные источники, библиотеки и помощь ИИ: всё из lab3 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py; образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine, haproxy:3.2.25-alpine). Новое: boto3/botocore 1.43.116 (S3-клиент), Pillow 12.3.0 (проверка JPEG), образ chrislusf/seaweedfs:3.80. Документация: AWS S3 (PutObject, x-amz-checksum-sha256, presigned URLs), SeaweedFS (`weed server -s3`, S3 API, статический конфиг identities с правами `Read/Write/List:`), спецификация PNG (структура чанков, CRC, IHDR, фильтры строк, Adam7), botocore Config (timeouts, `total_max_attempts`). При написании кода и тестов использовался ИИ-ассистент (Claude); весь код проверен и прогнан мной, объяснить и изменить его могу на защите. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверял на Windows 11, все команды в PowerShell. Версии Docker, Compose и ресурсы машины — в `evidence/stack.txt`. Python нужен только для тестов (стандартная библиотека, Pillow на хосте не нужен). Перед запуском остановить стенд lab3 (`docker compose down` в его папке), иначе занят порт 8080. + +```powershell +git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab4.git +cd mtusi2026lab4 +git checkout solution/bvt2303-babichenko +$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают +Copy-Item .env.example .env +python scripts/init_secrets.py # случайные пароли, S3-ключи и secrets/s3_config.json (папка в .gitignore) +docker compose up -d --build --wait # postgres, s3 -> migrate, s3-init -> 2 реплики api, janitor -> lb +docker compose ps -a # migrate и s3-init: Exited (0), остальные healthy/running + +python student-tests/collect_evidence.py # все проверки по очереди (~8 мин), вывод в evidence/, таблицы в этот отчёт +``` + +Отдельные проверки (make на моём Windows нет, ниже то же самое без make): + +| Команда из README | То же в PowerShell | +| --- | --- | +| `make check` | `python scripts/check_contract.py` | +| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` | +| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` | +| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` | +| тесты вложений (уровень 3–4) | `python student-tests/test_attachments.py` | +| S3 вне API, права ключа, bootstrap | `python student-tests/test_s3_security.py` | +| отказы S3/БД, janitor | `python student-tests/test_consistency.py [s3\|crash\|orphan\|aborted]` | +| 10+ одновременных загрузок, память | `python student-tests/upload_memory.py` | +| ручной проход очистки | `docker compose exec janitor python janitor.py --once --dry-run` | + +После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть её: `docker start ` или `docker compose up -d --wait`. На Linux/macOS вместо `.cmd` — `scripts/restart.sh` и `scripts/stop-one.sh`. + +Bootstrap: миграции применяет одноразовый сервис `migrate`, bucket и проверку прав S3 — одноразовый сервис `s3-init` (`s3_init.py`, тот же образ, что у API); оба запускаются сами при `docker compose up`, оба идемпотентны. Повторить вручную: `docker compose run --rm migrate`, `docker compose run --rm s3-init`. Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py` (новые пароли не подходят к старым volumes, поэтому удаляются вместе). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент --HTTP--> lb: HAProxy 3.2, 127.0.0.1:8080 (единственный опубликованный порт) + | round robin, без привязки клиента к реплике + +--> api-1 --+ + +--> api-2 --+--> postgres: пользователи, чаты, сообщения, МЕТАДАННЫЕ вложений (volume pgdata) + +--> redis: сессии (AOF, volume redisdata) + +--> s3 (SeaweedFS): БАЙТЫ вложений, приватный bucket (volume s3data) +janitor: раз в 5 мин удаляет брошенные загрузки и объекты-сироты (postgres + s3) +migrate: одноразово применяет migrations/*.sql +s3-init: одноразово создаёт приватный bucket и проверяет права ключей и закрытость bucket +сети: edge (lb <-> api) и data (всё остальное; internal, без выхода наружу) +``` + +| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` на 8404 | нет | edge; `127.0.0.1:8080`, `127.0.0.1:8404/stats` | не нужны | +| API ×2 | `api` (Dockerfile, `deploy.replicas: 2`) | python:3.11.13-slim | `/health/ready` (PostgreSQL + Redis) | нет | edge + data; наружу не публикуется | `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` | +| S3 | `s3` | chrislusf/seaweedfs:3.80 (`weed server -s3`: master, volume, filer и S3 в одном процессе) | S3-порт отвечает по HTTP (`wget`) | `s3data` (данные и метаданные filer) | только data; S3 8333, UI master 9333 и filer 8888 доступны только внутри стенда, наружу не публикуются | `s3_config` (конфиг ключей и прав) | +| Bootstrap S3 | `s3-init` (образ приложения, `s3_init.py`) | python:3.11.13-slim | завершается с кодом 0 | нет | data | admin-ключ + ключ API (для проверки прав) | +| Очистка | `janitor` (тот же образ) | python:3.11.13-slim | процесс работает | нет | data | `database_url` + ограниченный ключ S3 | +| Миграции | `migrate` | python:3.11.13-slim | завершается с кодом 0 | нет | data | только `database_url` | +| База данных | `postgres` | postgres:17.11-alpine | `pg_isready` | `pgdata` | data, 5432 | `POSTGRES_PASSWORD_FILE` | +| Сессии | `redis` | redis:7.4.11-alpine | `redis-cli ping` с паролем | `redisdata` (AOF) | data, 6379 | `include /run/secrets/redis_auth` | + +Почему SeaweedFS: из трёх разрешённых вариантов MinIO отпал — его образы больше не раздаются ни с Docker Hub, ни с quay.io (проверено при сборке стенда: `pull access denied` / `unauthorized`), а закреплённый тег без доступного образа не воспроизводится. У SeaweedFS образ на Docker Hub, а права ключей задаются статическим конфигом (`-s3.config`): отдельный ключ API с правами только на свой bucket получается без отдельной утилиты администрирования. Закреплена версия 3.80, `latest` не используется. + +Что перенесено из lab3 без изменений по смыслу: аутентификация и сессии в Redis, пользователи/чаты/сообщения, идемпотентность, HAProxy с проверкой готовности и аварийным режимом, двухшаговая остановка по SIGTERM, сервис `migrate`, свои тесты lab1–3. + +Что изменено в lab4: +- Новые файлы: `storage.py` (S3-клиент), `uploads.py` (приём и проверка файла), `attachments.py` (запись в две системы), `janitor.py` (очистка/сверка), `s3_init.py` (bootstrap S3), `migrations/004_attachments.sql`. +- `main.py`: endpoint загрузки, метаданных и скачивания; `attachment_ids` в сообщениях; `db.py`: вложения и связь сообщение–вложение. +- `compose.yaml`: сервисы `s3`, `s3-init`, `janitor`, volume `s3data`, пять новых секретов (admin-ключ, ключ API, конфиг ключей SeaweedFS); `infra/haproxy.cfg`: `timeout server 30s` (см. ниже); restart-скрипты перезапускают ещё `s3` и `janitor`. +- `scripts/init_secrets.py` дописывает недостающие секреты (в том числе `s3_config.json` с правами ключей) и не трогает существующие. + +### Модель данных + +`attachments` (миграция 004): `id`, `chat_id`, `owner_id`, `filename` (только для отображения), `content_type`, `size_bytes`, `sha256`, `width`, `height`, `object_key`, `state` (`uploading` | `ready`), `created_at`, `updated_at`. `object_key` = `originals/<32 случайных hex>`: его назначает сервер, имя файла в ключ не попадает (CHECK на формат). Внешний ключ `(chat_id, owner_id) -> chat_members`: загрузить можно только в свой чат. + +`message_attachments (message_id, chat_id, attachment_id, position)`: порядок вложений в сообщении. Оба внешних ключа составные и содержат один и тот же `chat_id`: `(message_id, chat_id) -> messages`, `(attachment_id, chat_id) -> attachments`. Поэтому вложение другого чата нельзя прикрепить даже прямым INSERT в обход API. Одно вложение можно прикрепить к нескольким сообщениям (контракт: «любой участник может ссылаться на доступное вложение чата»). Ограничение на текст сообщения ослаблено до `<= 16000 байт`: пустой текст допустим с вложением, проверку «текст или вложение» делает API. + +### Загрузка: что и в каком порядке проверяется + +`POST /api/v1/chats/{chat_id}/attachments?filename=...`, тело — сырые байты одного PNG/JPEG. + +1. До чтения тела: сессия (401) -> участник чата (404) -> `filename` 1–255 символов без `/`, `\`, NUL, управляющих символов, не `.`/`..` (400) -> `Content-Type` только `image/png`/`image/jpeg` (415) -> `Content-Length` больше 10 485 760 (413). При этих ошибках тело дочитывается и выбрасывается (до 20 MiB / 10 с, ничего не сохраняя), чтобы клиент получил ответ, а не обрыв соединения. +2. Тело читается кусками: размер и SHA-256 считаются на лету, данные пишутся во временный файл (до 256 KiB в памяти, остальное на диск контейнера). Первый байт сверх 10 MiB -> 413, чтение прекращается (работает и без `Content-Length`, при `Transfer-Encoding: chunked`). Таймауты: 10 с без данных, 60 с на всё тело (`UPLOAD_IDLE_TIMEOUT_SECONDS`, `UPLOAD_TIMEOUT_SECONDS`). +3. Тип по сигнатуре (`\x89PNG...`, `FF D8 FF`), не по заголовку. Не PNG/JPEG -> 415; заявленный тип не совпадает с содержимым -> 415. +4. Размер в пикселях читается из заголовка (IHDR / SOF) и проверяется ДО распаковки: больше 40 000 000 -> 413. +5. Проверка целостности с ограниченной памятью: + - PNG — своя потоковая проверка (`uploads.check_png`): CRC каждого чанка, порядок IHDR/PLTE/IDAT/IEND, допустимые тип цвета и глубина, ничего после IEND; IDAT распаковывается zlib кусками по 64 KiB (вывод не хранится), считается точный объём данных по IHDR (с учётом Adam7) и проверяется байт фильтра каждой строки. «Zip-бомба» не раздувает память: распаковка идёт порциями и обрывается, как только данных больше, чем обещал заголовок. + - JPEG — Pillow декодирует весь поток, но в масштабе 1/8 (`draft`): 40 Мпикс -> ~0,6 Мпикс, около 2 MiB. + - Одновременно в реплике проверяется не больше `DECODE_CONCURRENCY=2` файлов (CPU). + Битый файл -> 415. Любая ошибка разбора ловится и превращается в 415, а не 500. Проверка PNG сверена с Pillow: 400 случайных корректных PNG разных типов приняты, 400 испорченных (обрезка, перевёрнутый бит) отклонены. +6. Запись (см. следующий раздел) и ответ `201 Attachment` со `status=ready`, `variants=[]`, `error_code=null`. + +Временный файл удаляется в `finally` при любом исходе: оригинал после ответа существует только в S3. У контейнеров `api` нет volumes, это проверяет `test_consistency.py`. + +Скачивание: `GET /api/v1/attachments/{id}/content` — сначала проверка участника чата в PostgreSQL (иначе 404, как у несуществующего), потом GET из S3 и потоковая отдача кусками по 64 KiB с истинным `Content-Type`, `Content-Length`, `X-Content-Type-Options: nosniff`, `Content-Disposition: inline; filename*=UTF-8''...`. Без redirect и без подписанных URL. `variant` — только `original` (иное -> 400). + +### Согласованность PostgreSQL и S3 + +Распределённой транзакции нет, поэтому порядок записи выбран так, чтобы у каждого объекта в bucket была строка-указатель в БД (`attachments.py`): + +| Шаг | Если здесь отказ | +| --- | --- | +| 1. `INSERT attachments (state='uploading')` + COMMIT | в S3 ничего нет; клиенту 503 | +| 2. `PUT` объекта в S3 с `x-amz-checksum-sha256` (S3 сам сверяет SHA-256) | откат: короткий DELETE объекта (PUT мог пройти при таймауте) и удаление строки; если S3 недоступен и откат не удался, строка `uploading` остаётся указателем для janitor; клиенту 503 | +| 3. `UPDATE state='ready'` + COMMIT | клиенту 503; объект сразу НЕ удаляется: при обрыве связи COMMIT мог пройти, удаление сломало бы готовое вложение. Строка остаётся `uploading` -> её разберёт janitor | +| 4. `201` | только после шага 3 | + +Строки `uploading` не видны через API (GET -> 404) и не прикрепляются к сообщениям (404), сообщение может ссылаться только на `ready` (проверка под `FOR SHARE` в той же транзакции, что и INSERT сообщения). + +`janitor.py` (сервис `janitor`, раз в `JANITOR_INTERVAL_SECONDS=300`, один экземпляр за счёт `pg_try_advisory_lock`): +1. Строки `uploading` старше grace period: под `FOR UPDATE SKIP LOCKED` удалить объект (DELETE идемпотентен), затем строку. Если S3 недоступен — транзакция откатывается, строка остаётся до следующего прохода. +2. Обход bucket (`originals/`, постранично): объект старше grace period без строки в БД (в любом состоянии) — сирота, удаляется (перед удалением ещё раз проверяется, что строка не появилась). Живая загрузка всегда имеет строку, потому что строка коммитится до PUT. +3. Отчёт о `ready`-строках без объекта (`missing_objects`). Их janitor не удаляет: это потеря данных, её чинят восстановлением из резервной копии, а не автоматической очисткой; скачивание такого файла отвечает 503, не 500. + +Grace period: `ATTACHMENT_GRACE_SECONDS=3600`, janitor и API отказываются стартовать, если он меньше `2 x UPLOAD_TIMEOUT_SECONDS`. Самая долгая загрузка после записи строки живёт: PUT до 2 x (1 + 5) с + откат 2 с + UPDATE до 8 с, то есть меньше 30 с — в 120 раз меньше grace period. Поэтому janitor не может удалить загрузку, которая ещё идёт, а если бы реплика зависла дольше часа, её UPDATE не найдёт строку (`state='uploading'` уже удалена под блокировкой) и клиент получит 503, а не «ready без объекта». Есть `--dry-run` (только отчёт) и `--once`. + +Обрыв загрузки клиентом: строка пишется только после приёма и проверки всего тела, поэтому оборванная загрузка не оставляет ни строки, ни объекта; временный файл удаляется сразу. + +### Готовность и отказ S3 + +S3 не входит в `/health/ready`: без него не работают только загрузка и скачивание файлов, а вход, чаты, сообщения (в том числе со ссылкой на уже загруженные вложения) и метаданные вложений продолжают работать. Если бы readiness зависел от S3, отказ S3 вывел бы из балансировки все реплики и остановил бы весь мессенджер. При недоступном S3 файловые операции отвечают `503 unavailable` без подробностей. Критичны для readiness, как и в lab3, PostgreSQL и Redis. + +S3-клиент: connect timeout 1 с, read timeout 5 с, всего 2 попытки (`total_max_attempts`), откат — 1 попытка с таймаутом 2 с. Худший ответ при зависшем S3 около 13 с, поэтому в HAProxy `timeout server` увеличен с 15 до 30 с (пока клиент передаёт тело файла, этот таймаут не срабатывает: передачу ограничивают таймауты самого API). + +### Безопасность хранилища + +- Bucket приватный: в конфиге ключей SeaweedFS нет identity `anonymous`, поэтому любой запрос без подписи получает 403. Проверка вне API: `s3-init` и `test_s3_security.py` делают анонимные GET/HEAD/LIST/PUT прямо в S3 из внутренней сети — 403. +- Два ключа (`secrets/s3_config.json`, генерирует `scripts/init_secrets.py`): `admin` (Admin/Read/Write/List) есть только у `s3` (в конфиге) и `s3-init`; `messenger-api` для api и janitor — только `Read:messenger-attachments`, `Write:messenger-attachments`, `List:messenger-attachments`. Создать/удалить bucket, поменять политику или versioning ключ API не может — проверяют `s3-init` и `test_s3_security.py`. +- Порты S3 (8333), master (9333) и filer (8888) не публикуются, SeaweedFS только во внутренней сети `data` (`internal: true`); веб-интерфейсы master/filer доступны только контейнерам стенда, с хоста — connection refused. +- Bootstrap идемпотентен: `create_bucket` при существующем bucket получает `BucketAlreadyOwnedByYou/BucketAlreadyExists` и продолжает; проверочный объект ключа API создаётся и удаляется; в конце скрипт проверяет, что bucket private. Повторный `docker compose run --rm s3-init` ничего не ломает (тест). Права ключей — декларативный конфиг, применяется при старте `s3`. +- Приватный bucket и случайный ключ не заменяют проверку доступа в API: ключ объекта не секрет (может попасть в лог, бэкап, ошибку), а bucket «приватен» для всех, у кого нет ключа S3, — но не различает пользователей мессенджера. Кто участник какого чата, знает только API, поэтому проверка участника делается на каждый GET метаданных и байтов. + +### Соответствие `.env.example` + +| Переменная | Что делает в этом решении | +| --- | --- | +| APP_HOST, APP_PORT, APP_ENV | где слушает API (0.0.0.0:8080 внутри контейнера), APP_ENV пишется в лог | +| SESSION_TTL_SECONDS, SESSION_STORE=redis | срок жизни сессии 3600 с, сессии в Redis (как в lab3) | +| DATABASE_URL_FILE, CACHE_URL_FILE | пути к секретам с адресами PostgreSQL и Redis | +| API_REPLICAS | число реплик `api`, по умолчанию 2 | +| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) | +| S3_ENDPOINT, S3_REGION, S3_BUCKET | `http://s3:8333`, `us-east-1`, `messenger-attachments` (path-style адресация) | +| S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE | ограниченный ключ API (identity `messenger-api` в `s3_config.json`) | +| MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS | 10 485 760 байт и 40 000 000 пикселей | +| S3_CONNECT_TIMEOUT_SECONDS, S3_READ_TIMEOUT_SECONDS, S3_MAX_ATTEMPTS, S3_POOL_MAX | 1 с, 5 с, 2 попытки всего, 20 соединений | +| UPLOAD_TIMEOUT_SECONDS, UPLOAD_IDLE_TIMEOUT_SECONDS | 60 с на всё тело, 10 с без данных | +| DECODE_CONCURRENCY | одновременных проверок изображений в реплике, 2 | +| ATTACHMENT_GRACE_SECONDS, JANITOR_INTERVAL_SECONDS | grace period 3600 с, проход janitor раз в 300 с | +| DB_*, REDIS_*, SHUTDOWN_* | пулы, таймауты и остановка (как в lab3) | ## Доказательства критериев +Все файлы `evidence/*.txt` создаёт `python student-tests/collect_evidence.py` (вывод тестов как есть, с датой, commit и кодом выхода); сводка — `evidence/summary.md`. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS через балансировщик, с загрузкой, сообщением с `attachment_ids`, скачиванием и 404 постороннему | evidence/smoke.txt | +| Skeleton и контракт не изменены | `python scripts/check_contract.py` | PASS; `contracts/`, `tests/`, README не менялись | evidence/check.txt | +| 3: возможности lab3 сохранены | `test_balancing.py`, `test_idempotency.py`, `test_api.py`, `test_race.py`, `test_db.py`, `redis_info.py`, `explain.py`, `test_failures.py`, публичный failover | все PASS через балансировщик | evidence/balancing.txt, idempotency.txt, student-tests.txt, race.txt, db.txt, redis.txt, explain.txt, failures.txt, failover.txt | +| 3: S3 в общем стенде; метаданные в PostgreSQL, байты в приватном bucket, не в БД и не в каталоге API | `check_stack.py`, `test_s3_security.py`, `test_consistency.py aborted` | `s3` и `s3-init` в compose; у `api` нет volumes, временных файлов после запросов 0; анонимный доступ к bucket 403 | evidence/stack.txt, s3-security.txt, consistency.txt | +| 3: POST -> 201 ready после сохранения; оригинал побайтно; message с attachment_ids читается участниками; посторонний без доступа | smoke, `test_attachments.py` (roundtrip, access, message) | SHA-256 и размер совпадают, скачивает второй участник, посторонний 404 на метаданные и байты | evidence/smoke.txt, attachments.txt | +| 3: реальные PNG/JPEG ≤10 MiB и ≤40 Мпикс; MIME по содержимому; опасные filename; лимиты до неограниченного чтения/декодирования | `test_attachments.py` | ровно 10 MiB -> 201, +1 байт -> 413 (с Content-Length и chunked); 8000x5000 -> 201, 8001x5000 -> 413, «бомба» 100000x100000 -> 413 по заголовку; 13 опасных имён -> 400 | evidence/attachments.txt | +| 3: upload -> остановка обслужившей реплики -> download через другую | публичный `make test-failover` (lab4 загружает файл до отказа и скачивает после) для каждой реплики; `test_balancing.py` | PASS для обеих реплик; файл, загруженный через одну реплику, скачан через другую | evidence/failover.txt, balancing.txt | +| 3: restart с сохранением volumes сохраняет метаданные и оригинал | `run_persistence.py` (публичный persistence + ID/время старта контейнеров) | postgres, redis, s3, обе api, janitor, lb перезапущены (те же контейнеры, новое время старта); файл скачивается с прежним токеном | evidence/persistence.txt | +| 4: тесты: слишком большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; нет 500 | `test_attachments.py` | 413; 10 видов битых файлов -> 415; 7 подмен типа -> 415; неизвестный id и вложение другого чата (автор состоит в обоих) -> 404; дубли, 11 id -> 400; ни одного 500 | evidence/attachments.txt | +| 4: прерывание загрузки и отказ между S3/БД не оставляют вечных неучтённых объектов; механизм с grace period | `test_consistency.py` (aborted, crash, orphan, s3) | обрыв: строк и объектов не прибавилось; падение после PUT: строка `uploading` невидима, janitor не трогает её до grace, `--dry-run` только отчёт, затем удалены объект и строка, повтор — 0; сирота удаляется только после grace; `ready` не трогаются; missing — только отчёт | evidence/consistency.txt | +| 4: недоступность S3 | `test_consistency.py s3` (`docker compose stop s3` и `pause s3`) | файловые операции 503 без подробностей, остальное работает; после восстановления файлы на месте, мусор убран | evidence/consistency.txt | +| 4: лимиты/таймауты и потоковая обработка; ≥10 одновременных загрузок с пиком памяти | `upload_memory.py` | таблица ниже | evidence/upload-memory.txt, upload-memory.json | +| 4: bootstrap идемпотентен | `test_s3_security.py` (повторный `docker compose run --rm s3-init`) | exit 0, «bucket is private», старые файлы скачиваются, новая загрузка 201 | evidence/s3-security.txt | +| 4: отдельные S3 credentials с ограниченными правами | `test_s3_security.py`, `check_stack.py`, `s3-init` | ключ API: get/put/delete/list своего bucket — да; create/delete bucket, policy, versioning — отказ; admin-ключ и конфиг ключей только у s3 и s3-init | evidence/s3-security.txt, stack.txt | +| 4: консоль не открыта; проверка private bucket вне API | `test_s3_security.py`, логи `s3-init` | 127.0.0.1:8333/9333/8888 — connection refused, s3 только в сети data; анонимные GET/HEAD/LIST/PUT/policy прямо в S3 -> 403 | evidence/s3-security.txt | +| Выбранный трек 5 | — | не выбран | — | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотезы: +1. Память API-реплики почти не зависит от числа и размера одновременных загрузок: тело идёт во временный файл, PNG проверяется потоково, JPEG — в масштабе 1/8, проверок одновременно не больше 2 на реплику. Наивный вариант (тело в памяти + полное декодирование 40 Мпикс ~160 MiB) при 10 одновременных загрузках не поместился бы в лимит 512 MiB. +2. Отказ S3 затрагивает только операции с байтами файлов и не оставляет видимых или вечных «половинчатых» вложений. + +Методика памяти (`student-tests/upload_memory.py`): три сценария — 10 x PNG из случайного шума ~9,5 MiB (почти несжимаемый, максимум трафика); 10 x PNG 8000x5000 RGB (40 Мпикс, максимум распаковки); 20 смешанных. Перед каждым сценарием `docker compose restart api` (сбрасывает пик), прогрев 8 загрузками, затем все потоки стартуют одновременно (barrier) через балансировщик. Пик RSS — `VmHWM` из `/proc/1/status` процесса API (максимум за жизнь процесса по данным ядра, короткие всплески не теряются), дополнительно пик памяти cgroup контейнера и опрос раз в ~0,1–0,5 с. Проверяются коды 201 и SHA-256 скачанных файлов. Клиент — тот же компьютер, поэтому время загрузки ограничено и клиентом; это замер памяти, а не пропускной способности. + + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 | +| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 | +| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 | +| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 | +| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | + + +Вывод по памяти: рост RSS реплики при 10–20 одновременных загрузках — единицы MiB (см. таблицу), а не «размер файла x число загрузок»; верхняя оценка на загрузку — 256 KiB буфера временного файла + 64 KiB куска чтения + буферы boto3 при PUT; на проверку — 64 KiB распаковки PNG или ~2 MiB для JPEG, не больше 2 одновременно. Пик cgroup выше пика RSS, потому что в него входит page cache временных файлов (ядро освобождает его при нехватке памяти, OOM он не вызывает). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается отказ одной API-реплики и временная недоступность S3 (файловые операции 503, остальное работает). Не выдерживается отказ единственных `lb`, `postgres`, `redis`, `s3` и всей машины: стенд на одном узле. +- Повтор загрузки после потерянного ответа создаёт второе вложение (в контракте нет ключа идемпотентности для загрузки). Лишнее вложение, не прикреплённое к сообщению, остаётся: удаления в API нет; janitor удаляет только незавершённые загрузки. +- `missing_objects` janitor только находит; восстановление — из резервной копии S3, которой в стенде нет. +- Обход bucket janitor’ом держит в памяти список ключей (для учебного объёма — мегабайты); для миллионов объектов нужна сверка по диапазонам ключей или S3 Inventory. +- Временный файл загрузки (часть тела сверх 256 KiB) лежит на диске контейнера. В Linux `tempfile.TemporaryFile` создаётся уже без имени в каталоге (O_TMPFILE), поэтому даже при SIGKILL реплики место освобождается ядром, «забытых» файлов не бывает. Одновременно на диске может быть до 10 MiB на каждую идущую загрузку; лимита на число одновременных загрузок, кроме лимитов HAProxy/uvicorn, нет. +- Проверяется структура изображения, а не его безопасность для всех декодеров (например, JPEG с некорректными, но декодируемыми данными пройдёт). Перекодирование оригинала — задача lab5. +- MinIO собирался быть первым выбором, но его образы больше не доступны в публичных реестрах: закреплённый тег защищает от неожиданного обновления, но не от исчезновения образа. Для эксплуатации нужно зеркалить образы в свой реестр. SeaweedFS запущен одним процессом `weed server` на одном диске — это SPOF и не защита от потери диска; веб-интерфейсы master/filer без авторизации, закрыты только сетью. +- Скрипты `failover_metrics.py`, `rolling_update.py`, `test_expiry.py` из lab3 остались в `student-tests/` и работают со стендом, но в общий прогон lab4 не входят (это уровень 4 lab3, а не lab4). diff --git a/attachments.py b/attachments.py new file mode 100644 index 00000000..cb0595b3 --- /dev/null +++ b/attachments.py @@ -0,0 +1,72 @@ +"""Сохранение вложения: PostgreSQL (метаданные) + S3 (байты) без распределённой транзакции. + +Порядок записи и что происходит при отказе на каждом шаге: + + 1. INSERT attachments (state=uploading) и COMMIT — упало: в S3 ничего нет, клиенту 503. + 2. PUT объекта в S3 — упало: удаляем объект (на случай, если PUT + на самом деле прошёл) и строку, клиенту 503. + Если и это не вышло — остаётся uploading-строка, + её уберёт janitor после grace period. + 3. UPDATE state=ready и COMMIT — упало/неизвестно: клиенту 503; строка остаётся + uploading, объект есть -> janitor удалит оба. + Объект сразу НЕ удаляем: при обрыве связи COMMIT + мог пройти, и удаление сломало бы ready-вложение. + 4. 201 клиенту — только после шага 3. + +Клиенту видны и прикрепляемы только ready-строки, поэтому незавершённая загрузка ни на что не влияет. +""" +import logging +from uuid import UUID, uuid4 + +import db +import storage +from errors import unavailable +from uploads import ImageInfo, Upload + +log = logging.getLogger("messenger.attachments") + + +def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict: + attachment_id = uuid4() + key = storage.new_key(uuid4().hex) + db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type, + upload.size, upload.sha256, image.width, image.height, key) + try: + storage.put(key, upload.file, upload.size, image.content_type, upload.sha256) + except storage.StorageUnavailable: + compensate(attachment_id, key) + raise unavailable("File storage is temporarily unavailable") from None + row = db.finish_upload(attachment_id) + if row is None: + # janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace). + log.warning("Upload %s was reclaimed before it finished", attachment_id) + raise unavailable() + return row + + +def compensate(attachment_id: UUID, key: str) -> None: + """Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor.""" + try: + storage.delete(key, quick=True) + db.delete_upload_intent(attachment_id) + log.info("Upload %s rolled back after S3 failure", attachment_id) + except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту + log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__) + + +def public(row: dict) -> dict: + """Строка БД -> Attachment из контракта (без object_key и размеров в пикселях).""" + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": "ready", + "variants": [], + "error_code": None, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } diff --git a/auth.py b/auth.py new file mode 100644 index 00000000..c21212f8 --- /dev/null +++ b/auth.py @@ -0,0 +1,139 @@ +import base64 +import binascii +import hashlib +import logging +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from redis.backoff import NoBackoff +from redis.retry import Retry + +import settings +from errors import BASIC_CHALLENGE, unauthorized, unavailable + +log = logging.getLogger("messenger.auth") + +hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1) + +DUMMY_HASH = hasher.hash("there-is-no-such-user") + +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def password_matches(password_hash: str | None, password: str) -> bool: + try: + hasher.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """'Basic base64(username:password)' -> (username, password) или None. + + Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617). + """ + scheme, _, value = (header or "").strip().partition(" ") + if scheme.lower() != "basic" or not value or len(value) > 1024: + return None + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + username, colon, password = decoded.partition(":") + if not colon or not username or not username.isascii(): + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + scheme, _, value = (header or "").strip().partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value: + return None + return value + + + +KEY_PREFIX = "session:" + +_redis = redis.Redis( + connection_pool=redis.BlockingConnectionPool.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + max_connections=settings.REDIS_POOL_MAX, + timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) +) + + +@dataclass(frozen=True) +class Session: + user_id: UUID + key: str + + +def session_key(token: str) -> str: + return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: UUID) -> tuple[str, datetime]: + """Новая сессия: случайный токен и абсолютное время истечения.""" + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS) + try: + _redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000)) + except redis.RedisError as exc: + log.warning("Redis is unavailable on login: %s", type(exc).__name__) + raise unavailable() from None + return token, expires_at + + +def check_bearer(header: str | None) -> Session: + token = parse_bearer(header) + if token is None: + raise unauthorized("Bearer token is required") + key = session_key(token) + try: + user_id = _redis.get(key) + except redis.RedisError as exc: + log.warning("Redis is unavailable: %s", type(exc).__name__) + raise unavailable() from None + if user_id is None: + raise unauthorized("Session is invalid or expired") + return Session(UUID(user_id), key) + + +def delete_session(session: Session) -> None: + try: + _redis.delete(session.key) + except redis.RedisError: + raise unavailable() from None + + +def ping_redis() -> None: + try: + _redis.ping() + except redis.RedisError: + raise unavailable() from None + + +def login_failed(): + return unauthorized("Invalid username or password", BASIC_CHALLENGE) + + +def close() -> None: + _redis.close() + _redis.connection_pool.disconnect() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f7188716 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,211 @@ +name: messenger-lab4 + +x-app-build: &app-build + context: . + args: + APP_VERSION: ${APP_VERSION:-dev} + +x-s3-api-secrets: &s3-api-secrets + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + migrate: + build: *app-build + command: ["python", "migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений. + # Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config + # (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket. + # Анонимного доступа нет: identity "anonymous" в конфиге не объявлена. + s3: + image: chrislusf/seaweedfs:3.80 + entrypoint: ["/usr/bin/weed"] + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0", + "-master.volumeSizeLimitMB=1024", "-volume.max=0", + "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: ["s3data:/data"] + networks: [data] + healthcheck: + # S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт») + test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 15s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки. + s3-init: + build: *app-build + command: ["python", "s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + api: + build: *app-build + env_file: .env + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + <<: *s3-api-secrets + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "1.00", memory: 512M} + + # Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ. + janitor: + build: *app-build + command: ["python", "janitor.py"] + env_file: .env + environment: + CACHE_URL_FILE: "" # janitor не ходит в Redis + <<: *s3-api-secrets + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + healthcheck: + disable: true + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + + lb: + image: haproxy:3.2.25-alpine + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + postgres: + image: postgres:17.11-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] + secrets: [postgres_password] + volumes: ["pgdata:/var/lib/postgresql/data"] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + redis: + image: redis:7.4.11-alpine + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + - redisdata:/data + secrets: [redis_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + +networks: + edge: {} + data: + internal: true + +volumes: + pgdata: {} + redisdata: {} + s3data: {} + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + redis_auth: {file: ./secrets/redis_auth} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + s3_config: {file: ./secrets/s3_config.json} diff --git a/db.py b/db.py new file mode 100644 index 00000000..eb11ac25 --- /dev/null +++ b/db.py @@ -0,0 +1,359 @@ +import logging +import time +from contextlib import contextmanager +from datetime import datetime +from uuid import UUID, uuid4 + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +import settings +from errors import bad_request, conflict, not_found, unavailable + +log = logging.getLogger("messenger.db") + + +pool = ConnectionPool( + conninfo=settings.DATABASE_URL, + min_size=settings.DB_POOL_MIN, + max_size=settings.DB_POOL_MAX, + timeout=settings.DB_POOL_WAIT_SECONDS, + kwargs={ + "connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS, + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" + " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", + }, + check=ConnectionPool.check_connection, + open=False, +) + + +@contextmanager +def connection(wait_seconds: float | None = None): + try: + with pool.connection(timeout=wait_seconds) as conn: + yield conn + except (psycopg.OperationalError, PoolTimeout) as exc: + log.warning("PostgreSQL is unavailable: %s", type(exc).__name__) + raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text", "filename") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} + + +def ping() -> None: + with connection(wait_seconds=1) as conn: + conn.execute("SELECT 1") + + +MIGRATIONS_LOCK = 20260202 + + +def migration_files() -> list: + return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) + + +def run_migrations() -> list[str]: + files = migration_files() + applied = [] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now())""") + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + applied.append(path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) + return applied + + +def wait_and_migrate(max_wait_seconds: int = 60) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + applied = run_migrations() + log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") + return + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + log.warning("PostgreSQL is not ready yet, retry in 1 s") + time.sleep(1) + + +def missing_migrations() -> list[str]: + expected = [path.name for path in migration_files()] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + try: + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + except pg_errors.UndefinedTable: + return expected + return [name for name in expected if name not in done] + + +def wait_for_schema(max_wait_seconds: int) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + missing = missing_migrations() + except psycopg.OperationalError: + missing = None + if missing == []: + log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) + return + if time.monotonic() > deadline: + raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") + log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None + else "missing " + ", ".join(missing)) + time.sleep(1) + + + +USER_COLUMNS = "id, username, display_name, created_at" + + +def create_user(username: str, display_name: str, password_hash: str) -> dict: + try: + with connection() as conn: + return from_db(conn.execute( + f"""INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) + except pg_errors.UniqueViolation: + raise conflict("Username already exists") from None + + +def get_user(user_id: UUID) -> dict | None: + with connection() as conn: + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) + + +def get_user_with_password(username: str) -> dict | None: + with connection() as conn: + return from_db(conn.execute( + f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) + ).fetchone()) + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" + + +def create_chat(title: str, member_ids: list[UUID]) -> dict: + chat_id = uuid4() + members = sorted(member_ids) + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, to_db(title)), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in members], + ) + except pg_errors.ForeignKeyViolation: + raise bad_request("Unknown member") from None + return {**from_db(row), "member_ids": members} + + +def get_chat(chat_id: UUID, user_id: UUID) -> dict: + with connection() as conn: + chat = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s""", + (user_id, chat_id), + ).fetchone() + if chat is None: + raise not_found("Chat not found") + return from_db(chat) + + +def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" + where, params = "", [user_id] + if after is not None: + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params += list(after) + with connection() as conn: + rows = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + {where} + ORDER BY c.created_at, c.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids""" + + +def is_member(conn, chat_id: UUID, user_id: UUID) -> bool: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() is not None + + +def check_member(chat_id: UUID, user_id: UUID) -> None: + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + + +def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID], + idempotency_key: str, request_hash: bytes) -> dict: + try: + with connection() as conn: + with conn.transaction(): + if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404 + raise not_found("Chat not found") + if attachment_ids: + # Только готовые вложения ЭТОГО чата. FOR SHARE: строку не удалят до конца транзакции. + found = conn.execute( + """SELECT id FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state = 'ready' FOR SHARE""", + (attachment_ids, chat_id), + ).fetchall() + if len(found) != len(attachment_ids): + raise not_found("Attachment not found") + created = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), + ).fetchone() + if created is not None: + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + """INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES (%s, %s, %s, %s)""", + [(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)], + ) + return {**from_db(created), "attachment_ids": list(attachment_ids)} + saved = conn.execute( + f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m + WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""", + (chat_id, sender_id, idempotency_key), + ).fetchone() + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен + raise unavailable() + if bytes(saved.pop("request_hash")) != request_hash: + raise conflict("Idempotency-Key was already used with a different request") + return from_db(saved) + + +def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """История чата после курсора (keyset-пагинация), до limit+1 строк.""" + where, params = "m.chat_id = %s", [chat_id] + if after is not None: + where += " AND (m.created_at, m.id) > (%s, %s)" + params += list(after) + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + rows = conn.execute( + f"""SELECT {MESSAGE_COLUMNS} FROM messages m + WHERE {where} + ORDER BY m.created_at, m.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + +# ---------- вложения ---------- + +ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "created_at, updated_at, object_key") + + +def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, + size_bytes: int, sha256: str, width: int, height: int, object_key: str) -> None: + """Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в S3. + + Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт + и удалит его, если загрузка не дошла до ready (обрыв, падение реплики, отказ S3 или БД). + """ + try: + with connection() as conn: + conn.execute( + """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, object_key, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""", + (attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256, + width, height, object_key), + ) + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + + +def finish_upload(attachment_id: UUID) -> dict | None: + """Шаг 3 из 3: объект в S3 уже сохранён -> ready. None, если строку успел убрать janitor.""" + with connection() as conn: + row = conn.execute( + f"""UPDATE attachments SET state = 'ready', updated_at = now() + WHERE id = %s AND state = 'uploading' + RETURNING {ATTACHMENT_COLUMNS}""", + (attachment_id,), + ).fetchone() + return from_db(row) + + +def delete_upload_intent(attachment_id: UUID) -> None: + with connection(wait_seconds=1) as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,)) + + +def get_attachment(attachment_id: UUID, user_id: UUID) -> dict: + """Готовое вложение, если user_id — участник его чата. Иначе 404 (не раскрываем существование).""" + with connection() as conn: + row = conn.execute( + f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))} + FROM attachments a + JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s + WHERE a.id = %s AND a.state = 'ready'""", + (user_id, attachment_id), + ).fetchone() + if row is None: + raise not_found("Attachment not found") + return from_db(row) diff --git a/errors.py b/errors.py new file mode 100644 index 00000000..a6414768 --- /dev/null +++ b/errors.py @@ -0,0 +1,40 @@ +# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт. +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions +BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "Service temporarily unavailable") -> ApiError: + return ApiError(503, "unavailable", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 00000000..dc0ed88a --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,67 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-10-21 21:03:05, commit: 2dbed80 +# command: python.exe student-tests/test_attachments.py +# exit code: 0, 4.0 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 201 +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 201 5ffe6847 +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200 +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +11 passed, 0 failed diff --git a/evidence/balancing.txt b/evidence/balancing.txt new file mode 100644 index 00000000..dbbf431c --- /dev/null +++ b/evidence/balancing.txt @@ -0,0 +1,18 @@ +# balancing: balancing, shared sessions/data/files, X-Instance-Id +# date: 2026-10-21 21:08:50, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_balancing.py +# exit code: 0, 1.9 s + +20 x GET /users/me with one token -> replicas: {'6128eb268c44': 10, '11a93acecaca': 10} +PASS: one session works on every replica, no second login (sessions are shared in Redis) +message written by 11a93acecaca; read back by ['11a93acecaca', '6128eb268c44'] +PASS: a message written via one replica is read right away via another (shared PostgreSQL) +logout on 6128eb268c44; old token then rejected by ['11a93acecaca', '6128eb268c44'] +PASS: logout on one replica revokes the session on all of them +first POST on 6128eb268c44 -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af; same key on 11a93acecaca -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af +PASS: Idempotency-Key is shared: a retry through another replica returns the same message +file uploaded via 6128eb268c44; downloaded byte for byte via ['11a93acecaca', '6128eb268c44'] +PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local) +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['11a93acecaca', '6128eb268c44']; api containers: ['11a93acecaca', '6128eb268c44'] +PASS: X-Instance-Id is in every response and points to a real container +6 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..61c52c64 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-10-21 21:02:57, commit: 2dbed80 +# command: python.exe scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 00000000..b345844f --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,47 @@ +# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage +# date: 2026-10-21 21:08:14, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_consistency.py +# exit code: 0, 148.7 s + +declared 9 000 000 bytes, sent 211817, closed the connection +declared 9 000 000 bytes, sent 3111817, closed the connection +rows in attachments: 32 -> 32; objects in bucket: 31 -> 31 +temporary upload files in api containers: {'11a93acecaca': 0, '6128eb268c44': 0} (removed when the request ends) +api containers have no volumes: the original is never stored locally +PASS: an interrupted upload leaves no row, no object and no local file +crash after S3 PUT: row 04aacbf6 state=uploading, object originals/219175ec1d5647f0aeaec84ee8243cb3 exists in S3 +the half-written attachment is invisible: GET -> 404, attach to a message -> 404 +janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.101} -> fresh row and object are kept (the upload might still be running) +row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.092} -> nothing deleted +janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.096} -> row and object deleted +janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.103} -> idempotent +the ready attachment of the same chat is untouched +PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period +object originals/867f5c5d42bf4bf9973b60ef8856d88f put into the bucket without a row in PostgreSQL +janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.102} -> fresh object kept +after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.098} -> reported, kept +janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.1} -> orphan deleted +ready attachments older than the grace period are untouched (they have rows) +object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500) +PASS: orphans are found and deleted only after the grace period; missing objects are reported +--- S3 stopped (connection refused / name not resolved): docker compose stop s3 +upload -> 503 unavailable in 20141 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose start s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 35, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.099} +PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up +--- S3 frozen (no answer, timeouts): docker compose pause s3 +upload -> 503 unavailable in 14766 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose unpause s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 37, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.094} +PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up +5 passed, 0 failed diff --git a/evidence/db.txt b/evidence/db.txt new file mode 100644 index 00000000..481e4035 --- /dev/null +++ b/evidence/db.txt @@ -0,0 +1,21 @@ +# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations +# date: 2026-10-21 21:09:10, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_db.py +# exit code: 0, 14.2 s + +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 57 -> 58 +PASS: chat row was inserted and then rolled back together with members +direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" +PASS: the database itself rejects a message from a non-member +stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97) +PASS: password is stored as Argon2id hash +unique constraint on users: UNIQUE (username) +PASS: username is unique in the database +applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; running the migrate service once more +after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; log: Migrations applied: none (schema is up to date) +PASS: migrations are applied once, a repeated run does nothing +api replicas that checked the schema at start: 8; api log lines 'Migrations applied': 0; migrate service: Exited (0) 6 minutes ago +PASS: only the migrate service applies migrations, replicas wait for the schema +unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) +PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database +7 passed, 0 failed diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 00000000..678f4735 --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,99 @@ +=== Data volume + messages_total | messages_in_big_chat +----------------+---------------------- + 20146 | 15000 + + +=== History index + indexdef +---------------------------------------------------------------------------------------- + CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id) + + +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + QUERY PLAN +------------------------------------------------------------------------------------------------ + Limit (actual time=0.005..0.011 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) + Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) + Buffers: shared hit=5 + Planning: + Buffers: shared hit=6 + Planning Time: 0.060 ms + Execution Time: 0.020 ms + + +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.005..0.012 rows=51 loops=1) + Buffers: shared hit=6 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) + Buffers: shared hit=6 + Planning Time: 0.044 ms + Execution Time: 0.031 ms + + +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.020..0.225 rows=51 loops=1) + Buffers: shared hit=60 + -> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1) + Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) + Buffers: shared hit=60 + SubPlan 1 + -> Sort (actual time=0.003..0.003 rows=0 loops=51) + Sort Key: a."position" + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Recheck Cond: (message_id = m.id) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Index Cond: (message_id = m.id) + Buffers: shared hit=51 + Planning: + Buffers: shared hit=63 + Planning Time: 0.157 ms + Execution Time: 0.242 ms + + +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Limit (actual time=0.878..0.884 rows=51 loops=1) + Buffers: shared hit=258 + -> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1) + Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) + Buffers: shared hit=258 + Planning Time: 0.037 ms + Execution Time: 0.897 ms + + +=== 4. Query 2 without the index (index is dropped only inside this transaction) + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual time=1.097..1.101 rows=51 loops=1) + Buffers: shared hit=261 + -> Sort (actual time=1.097..1.098 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 31kB + Buffers: shared hit=261 + -> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1) + Filter: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) + Rows Removed by Filter: 15146 + Buffers: shared hit=261 + Planning: + Buffers: shared hit=4 + Planning Time: 0.051 ms + Execution Time: 1.115 ms + + +=== ROLLBACK: test data removed, index is back + explain_users_left +-------------------- + 0 + diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 00000000..597939b2 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,43 @@ +# explain: EXPLAIN of the history page (full plans: explain-history.txt) +# date: 2026-10-21 21:09:14, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/explain.py +# exit code: 0, 1.1 s + +=== Data volume +=== History index +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) + Buffers: shared hit=5 + Buffers: shared hit=6 + Execution Time: 0.020 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + Buffers: shared hit=6 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Buffers: shared hit=6 + Execution Time: 0.031 ms +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + Buffers: shared hit=60 + -> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1) + Buffers: shared hit=60 + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + Buffers: shared hit=63 + Execution Time: 0.242 ms +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + Buffers: shared hit=258 + -> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1) + Buffers: shared hit=258 + Execution Time: 0.897 ms +=== 4. Query 2 without the index (index is dropped only inside this transaction) + Buffers: shared hit=261 + Buffers: shared hit=261 + -> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1) + Buffers: shared hit=261 + Buffers: shared hit=4 + Execution Time: 1.115 ms +=== ROLLBACK: test data removed, index is back +PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..9bb22e80 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,33 @@ +# failover: public failover test for every replica, file downloaded via the other one (make test-failover) +# date: 2026-10-21 21:09:54, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/run_failover.py +# exit code: 0, 39.7 s + +hook: scripts\stop-one.cmd; api replicas: ['11a93acecaca', '6128eb268c44'] +=== run 1: api containers before +SERVICE CONTAINER ID STATUS +api 11a93acecaca Up 36 seconds (healthy) +api 6128eb268c44 Up 36 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.39} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 11a93acecaca Exited (0) 11 seconds ago +api 6128eb268c44 Up 53 seconds (healthy) +the hook stopped 11a93acecaca (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 11a93acecaca -> healthy and back in HAProxy +=== run 2: api containers before +SERVICE CONTAINER ID STATUS +api 11a93acecaca Up 2 seconds (healthy) +api 6128eb268c44 Up 56 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 4.328} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 11a93acecaca Up 18 seconds (healthy) +api 6128eb268c44 Exited (0) 11 seconds ago +the hook stopped 6128eb268c44 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 6128eb268c44 -> healthy and back in HAProxy +PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 00000000..28d1d694 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,29 @@ +# failures: Redis and PostgreSQL outages via the balancer +# date: 2026-10-21 21:11:20, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_failures.py +# exit code: 0, 55.6 s + +stopping redis +/health/live -> 200, /health/ready -> 503 (8016 ms) +valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked +unknown token -> 503, not 200: nobody is trusted without a check +no token -> 401 +login -> 503: there is nowhere to store the session +registration -> 201: it needs only PostgreSQL +starting redis +after start: the old token works, confirmed messages are in place +PASS: Redis outage gives 503 and does not let anyone in without a session check +killing redis with SIGKILL (docker compose kill redis): no graceful shutdown +right after the crash: GET /users/me -> 503 +starting redis +after start: the session issued before the crash still works (AOF, appendfsync always) +PASS: sessions survive a Redis crash, not only a clean restart +stopping postgres +/health/live -> 200, /health/ready -> 503 (1016 ms) +valid token: users/me, chats, history, new message -> 503 unavailable, no details +no token -> 401, unknown token -> 401: anonymous users are not let in +login -> 503, registration -> 503 +starting postgres +after start: the old token works, confirmed messages are in place +PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out +3 passed, 0 failed diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 00000000..205a798d --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-10-21 21:08:52, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_idempotency.py +# exit code: 0, 2.4 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'6128eb268c44': 10, '11a93acecaca': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..c4db3afc --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 4, + "successful_requests": 7379, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 245.844, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 00000000..8529586e --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-10-21 21:10:24, commit: 2dbed80 + uncommitted changes +# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.4 s + +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 4, + "successful_requests": 7379, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 245.844, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..f23f416b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,24 @@ +# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence) +# date: 2026-10-21 21:12:49, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/run_persistence.py +# exit code: 0, 40.2 s + +before: container, ID, started at + messenger-lab4-api-1 11a93acecaca 2026-10-21T18:11:37.067348364Z + messenger-lab4-api-2 6128eb268c44 2026-10-21T18:11:36.916459116Z + messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:11:33.313531682Z + messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:11:40.981519931Z + messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:11:22.482229396Z + messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:11:22.626999402Z + messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:11:32.264154477Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after (hook + waiting for readiness took 39.2 s): container, ID, started at, restarted? + messenger-lab4-api-1 11a93acecaca 2026-10-21T18:12:46.288933033Z same container: True, restarted: True + messenger-lab4-api-2 6128eb268c44 2026-10-21T18:12:46.174283308Z same container: True, restarted: True + messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:12:42.503993973Z same container: True, restarted: True + messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:12:47.684809415Z same container: True, restarted: True + messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:12:11.527270274Z same container: True, restarted: True + messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:12:11.368416107Z same container: True, restarted: True + messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:12:21.264135454Z same container: True, restarted: True +PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, new start time); data, the attachment bytes and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 00000000..7e5c4dfd --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-10-21 21:08:56, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_race.py +# exit code: 0, 1.2 s + +sent 100 messages from 10 clients in 0.22 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/redis.txt b/evidence/redis.txt new file mode 100644 index 00000000..b118ac4e --- /dev/null +++ b/evidence/redis.txt @@ -0,0 +1,14 @@ +# redis: Redis as the session store +# date: 2026-10-21 21:09:13, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/redis_info.py +# exit code: 0, 2.8 s + +session key: session:f4f1fe0235d139de...; value is the user id: True +TTL of the key, seconds: 3599 +sessions in Redis: 126 +config appendonly = yes +config appendfsync = always +config maxmemory-policy = noeviction +aof_enabled:1 +aof_last_write_status:ok +without password: NOAUTH Authentication required. diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt new file mode 100644 index 00000000..d118d616 --- /dev/null +++ b/evidence/s3-security.txt @@ -0,0 +1,44 @@ +# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap +# date: 2026-10-21 21:03:14, commit: 2dbed80 +# command: python.exe student-tests/test_s3_security.py +# exit code: 0, 9.5 s + +anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous head_object directly to S3 (from the data network) -> 403 +anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied +anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied +the same file through the API for a chat member -> 200 +PASS: the bucket is private: no anonymous read, list, write or policy read +API key: list buckets -> ['messenger-attachments'] +API key: get own object -> allowed +API key: put object -> allowed +API key: delete object -> allowed +API key: list bucket -> allowed +API key: create another bucket -> denied: AccessDenied +API key: make bucket public (put policy) -> denied: NotImplemented +API key: read bucket policy -> denied: NoSuchBucketPolicy +API key: delete bucket -> denied: AccessDenied +API key: change versioning -> denied: NotImplemented +PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied +compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand) +127.0.0.1:8333 from the host -> connection refused +127.0.0.1:9333 from the host -> connection refused +127.0.0.1:8888 from the host -> connection refused +127.0.0.1:19333 from the host -> connection refused +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init +s3-init: s3-init: bucket messenger-attachments exists +s3-init: s3-init: api key can put/get/list/delete objects of its bucket +s3-init: s3-init: api key cannot create buckets (AccessDenied) +s3-init: s3-init: bucket is private (anonymous list/get/put -> [403]) +after the repeated bootstrap: old file downloads, new upload works +PASS: bucket/user/policy bootstrap can be repeated safely +4 passed, 0 failed diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..95abed91 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test), with attachment +# date: 2026-10-21 21:03:01, commit: 2dbed80 +# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 0.6 s + +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 00000000..9b6415d5 --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,40 @@ +# stack: stand layout: replicas, ports, limits, networks, secrets, S3 +# date: 2026-10-21 21:03:00, commit: 2dbed80 +# command: python.exe student-tests/check_stack.py +# exit code: 0, 3.0 s + +host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 +Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop + +SERVICE CONTAINER ID STATUS PORTS +api 11a93acecaca Up 20 seconds (healthy) 8080/tcp +api 6128eb268c44 Up 20 seconds (healthy) 8080/tcp +janitor baefc0e97dff Up 20 seconds +lb 03697e5fe34c Up 18 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +migrate aca090330db4 Exited (0) 22 seconds ago +postgres f7a6a1ce25a7 Up 2 minutes (healthy) +redis b4523b39a0fe Up 2 minutes (healthy) +s3 e7cbe6ddfa4a Up 2 minutes (healthy) +s3-init 60719e674e18 Exited (0) 21 seconds ago + +container image cpus memory host ports networks +messenger-lab4-api-1 messenger-lab4-api 1.00 512 MiB - data,edge +messenger-lab4-api-2 messenger-lab4-api 1.00 512 MiB - data,edge +messenger-lab4-janitor-1 messenger-lab4-janitor 0.25 192 MiB - data +messenger-lab4-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge +messenger-lab4-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data +messenger-lab4-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data +messenger-lab4-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data + +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] + +through http://127.0.0.1:8080 one session is served by 2 replicas: ['11a93acecaca', '6128eb268c44'] +/health/ready -> 200 from 11a93acecaca +PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor only in the internal data network, S3 admin keys only in s3 and s3-init diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..9ac2c0fe --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-10-21 21:08:55, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/test_api.py +# exit code: 0, 2.7 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/summary.md b/evidence/summary.md new file mode 100644 index 00000000..5df443a4 --- /dev/null +++ b/evidence/summary.md @@ -0,0 +1,40 @@ +# Сводка проверок + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 40 | evidence/persistence.txt | + +# Замеры + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 | +| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 | +| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 | +| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 | +| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..30501787 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,124 @@ +{ + "date": "2026-10-21 21:08", + "api_limit_mib": 512, + "api_limit_cpus": 1.0, + "runs": [ + { + "scenario": "10 x ~9.5 MiB PNG (noise)", + "concurrency": 10, + "file_bytes": [ + 9953047 + ], + "total_mib": 94.9, + "all_201": true, + "wall_seconds": 1.52, + "latency_s": { + "p50": 1.38, + "max": 1.52 + }, + "sampler_rounds": 9, + "replicas": [ + { + "replica": "11a93acecaca", + "uploads_served": 5, + "rss_before_mib": 93.5, + "rss_peak_mib": 106.2, + "rss_growth_mib": 12.6, + "rss_peak_sampled_mib": 106.2, + "cgroup_peak_mib": 134.1, + "cgroup_peak_sampled_mib": 127.7 + }, + { + "replica": "6128eb268c44", + "uploads_served": 5, + "rss_before_mib": 94.8, + "rss_peak_mib": 105.3, + "rss_growth_mib": 10.5, + "rss_peak_sampled_mib": 105.3, + "cgroup_peak_mib": 135.2, + "cgroup_peak_sampled_mib": 131.4 + } + ] + }, + { + "scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", + "concurrency": 10, + "file_bytes": [ + 116714 + ], + "total_mib": 1.1, + "all_201": true, + "wall_seconds": 1.09, + "latency_s": { + "p50": 0.85, + "max": 1.09 + }, + "sampler_rounds": 8, + "replicas": [ + { + "replica": "11a93acecaca", + "uploads_served": 5, + "rss_before_mib": 93.2, + "rss_peak_mib": 96.6, + "rss_growth_mib": 3.4, + "rss_peak_sampled_mib": 96.6, + "cgroup_peak_mib": 84.3, + "cgroup_peak_sampled_mib": 77.3 + }, + { + "replica": "6128eb268c44", + "uploads_served": 5, + "rss_before_mib": 93.1, + "rss_peak_mib": 94.7, + "rss_growth_mib": 1.6, + "rss_peak_sampled_mib": 94.6, + "cgroup_peak_mib": 86.8, + "cgroup_peak_sampled_mib": 77.3 + } + ] + }, + { + "scenario": "20 mixed: large + 40 Mpx + JPEG", + "concurrency": 20, + "file_bytes": [ + 1492, + 116714, + 9953047 + ], + "total_mib": 133.2, + "all_201": true, + "wall_seconds": 1.77, + "latency_s": { + "p50": 1.37, + "max": 1.77 + }, + "sampler_rounds": 10, + "replicas": [ + { + "replica": "11a93acecaca", + "uploads_served": 10, + "rss_before_mib": 94.8, + "rss_peak_mib": 121.7, + "rss_growth_mib": 26.9, + "rss_peak_sampled_mib": 121.7, + "cgroup_peak_mib": 181.9, + "cgroup_peak_sampled_mib": 180.6 + }, + { + "replica": "6128eb268c44", + "uploads_served": 10, + "rss_before_mib": 95.0, + "rss_peak_mib": 116.1, + "rss_growth_mib": 21.0, + "rss_peak_sampled_mib": 116.1, + "cgroup_peak_mib": 155.6, + "cgroup_peak_sampled_mib": 155.6 + } + ] + } + ], + "single_file_extra_rss_mib": { + "naive": 161, + "streaming": 0 + } +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 00000000..cdcb73fd --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,13 @@ +# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas +# date: 2026-10-21 21:08:48, commit: 2dbed80 + uncommitted changes +# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json +# exit code: 0, 34.2 s + +client: Python 3.11.1 on Windows 10 +preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)... +large: 9953047 bytes; 40 Mpx: 116714 bytes +{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_201": true, "wall_seconds": 1.52, "latency_s": {"p50": 1.38, "max": 1.52}, "sampler_rounds": 9, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.5, "rss_peak_mib": 106.2, "rss_growth_mib": 12.6, "rss_peak_sampled_mib": 106.2, "cgroup_peak_mib": 134.1, "cgroup_peak_sampled_mib": 127.7}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 94.8, "rss_peak_mib": 105.3, "rss_growth_mib": 10.5, "rss_peak_sampled_mib": 105.3, "cgroup_peak_mib": 135.2, "cgroup_peak_sampled_mib": 131.4}]} +{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_201": true, "wall_seconds": 1.09, "latency_s": {"p50": 0.85, "max": 1.09}, "sampler_rounds": 8, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.2, "rss_peak_mib": 96.6, "rss_growth_mib": 3.4, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 84.3, "cgroup_peak_sampled_mib": 77.3}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 93.1, "rss_peak_mib": 94.7, "rss_growth_mib": 1.6, "rss_peak_sampled_mib": 94.6, "cgroup_peak_mib": 86.8, "cgroup_peak_sampled_mib": 77.3}]} +{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_201": true, "wall_seconds": 1.77, "latency_s": {"p50": 1.37, "max": 1.77}, "sampler_rounds": 10, "replicas": [{"replica": "11a93acecaca", "uploads_served": 10, "rss_before_mib": 94.8, "rss_peak_mib": 121.7, "rss_growth_mib": 26.9, "rss_peak_sampled_mib": 121.7, "cgroup_peak_mib": 181.9, "cgroup_peak_sampled_mib": 180.6}, {"replica": "6128eb268c44", "uploads_served": 10, "rss_before_mib": 95.0, "rss_peak_mib": 116.1, "rss_growth_mib": 21.0, "rss_peak_sampled_mib": 116.1, "cgroup_peak_mib": 155.6, "cgroup_peak_sampled_mib": 155.6}]} +one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB +PASS: all uploads succeeded; largest RSS growth of one replica: 26.9 MiB (limit 512 MiB); results in evidence/upload-memory.json diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..1605eed3 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,77 @@ +# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики). +# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить +# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky). + +global + log stdout format raw local0 info + maxconn 4096 + # При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с. + hard-stop-after 10s + +# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик. +# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s # перечитывать список реплик раз в 2 с + hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlog-normal # в лог только ошибки, а не каждый запрос + timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос + timeout client 30s + # Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка + # при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт, + # сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT). + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода. + # Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key. + retries 2 + option redispatch 1 + +frontend api_in + bind :8080 + option forwardfor + default_backend api + +backend api + balance roundrobin + # Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis. + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT. + # Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP. + # observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки. + server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + # Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis + # или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате + # контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает. + # Пока готова хоть одна реплика, эти записи не используются. + # allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend). + server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + option allbackups + +# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1). +# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда. +frontend stats + bind :8404 + monitor-uri /lb-health + monitor fail if { nbsrv(api) lt 1 } + stats enable + stats uri /stats + stats refresh 2s diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 00000000..33c8c087 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,13 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +appendonly yes +appendfsync always +save "" + +maxmemory 64mb +maxmemory-policy noeviction + +include /run/secrets/redis_auth diff --git a/infra/s3-init.sh b/infra/s3-init.sh new file mode 100644 index 00000000..ab0adaee --- /dev/null +++ b/infra/s3-init.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# Bootstrap of the S3 storage (MinIO), run by the one-shot service s3-init before api starts. +# Idempotent: every step can be repeated (docker compose run --rm s3-init) without losing data. +# 1. private bucket S3_BUCKET (no anonymous access, versioning off); +# 2. policy messenger-api: only Get/Put/Delete/List objects of this bucket (JSON below); +# 3. separate user for api/janitor with this policy. Root credentials stay in s3 and s3-init only. +# Secrets are read from /run/secrets and are never printed. +set -eu + +BUCKET="${S3_BUCKET:?S3_BUCKET is not set}" +ROOT_USER="$(cat /run/secrets/s3_root_user)" +ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)" +API_KEY="$(cat /run/secrets/s3_access_key_id)" +API_SECRET="$(cat /run/secrets/s3_secret_access_key)" +POLICY=messenger-api +export MC_CONFIG_DIR=/tmp/mc + +# Least privilege for the api key: objects of ONE bucket. No bucket admin, policies, users or other buckets. +cat > /tmp/policy.json </dev/null 2>&1; do + i=$((i + 1)) + [ "$i" -ge 30 ] && { echo "s3-init: MinIO is not reachable"; exit 1; } + sleep 1 +done + +mc mb --ignore-existing "local/${BUCKET}" >/dev/null +mc anonymous set none "local/${BUCKET}" >/dev/null +echo "s3-init: bucket ${BUCKET} exists" + +mc admin policy create local "$POLICY" /tmp/policy.json >/dev/null +echo "s3-init: policy ${POLICY} created/updated" + +mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null +info="$(mc admin user info local "$API_KEY" 2>/dev/null || true)" +case "$info" in + *"$POLICY"*) ;; + *) mc admin policy attach local "$POLICY" --user "$API_KEY" >/dev/null ;; +esac +echo "s3-init: api user exists with policy ${POLICY} only" + +access="$(mc anonymous get "local/${BUCKET}")" +case "$access" in + *private*) echo "s3-init: bucket is private (anonymous access: none)" ;; + *) echo "s3-init: bucket is NOT private: $access"; exit 1 ;; +esac diff --git a/janitor.py b/janitor.py new file mode 100644 index 00000000..58bee8f1 --- /dev/null +++ b/janitor.py @@ -0,0 +1,162 @@ +"""Очистка незавершённых загрузок и сверка PostgreSQL <-> S3. + +Запускается отдельным сервисом `janitor` (раз в JANITOR_INTERVAL_SECONDS) или вручную: + + python janitor.py --once [--dry-run] [--grace-seconds N] + +Проход состоит из двух шагов. Оба трогают только то, что старше grace period +(ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч, минимум 2 x UPLOAD_TIMEOUT_SECONDS): загрузка, +которая ещё идёт, столько не живёт, поэтому свежие и готовые вложения не удаляются. + +1. Брошенные намерения: строки state=uploading старше grace. Для каждой (под FOR UPDATE SKIP LOCKED) + удалить объект в S3 (идемпотентно) и затем строку. Если S3 недоступен, строка остаётся + до следующего прохода — указатель на объект не теряется. +2. Сирота в S3: объект под originals/ старше grace, для которого в БД нет строки ни в каком + состоянии. API кладёт объект только после COMMIT строки-намерения, поэтому у живой загрузки + строка всегда есть. Удаляется. + Заодно отчёт: ready-строки, чьего объекта нет в bucket (missing). Их не трогаем — это потеря + данных, её разбирает человек (восстановление из резервной копии), а не автоматическая очистка. + +Одновременно работает только один janitor: pg_try_advisory_lock. +""" +import argparse +import json +import logging +import signal +import threading +import time +from datetime import datetime, timedelta, timezone + +import psycopg +from psycopg.rows import dict_row + +import settings +import storage + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.janitor") + +JANITOR_LOCK = 20260404 +BATCH = 100 +STOP = threading.Event() + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]: + """Шаг 1. Возвращает (найдено, удалено).""" + found = deleted = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, object_key FROM attachments + WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + FOR UPDATE SKIP LOCKED""", + (grace, BATCH), + ).fetchall() + found += len(rows) + if dry_run or not rows: + return found, deleted + for row in rows: + storage.delete(row["object_key"]) # недоступен S3 -> исключение, транзакция откатится + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],)) + deleted += 1 + log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"]) + if len(rows) < BATCH: + return found, deleted + return found, deleted + + +def sweep_objects(conn, grace: int, dry_run: bool) -> dict: + """Шаг 2. Обход bucket постранично; сверка ключей с БД пачками по 1000.""" + started = datetime.now(timezone.utc) + cutoff = started - timedelta(seconds=grace) + scanned = orphans = orphans_deleted = 0 + listed = set() + page: list[tuple[str, datetime]] = [] + + def check(page_items): + nonlocal orphans, orphans_deleted + keys = [key for key, _ in page_items] + known = {r["object_key"] for r in conn.execute( + "SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,)).fetchall()} + for key, modified in page_items: + if key in known or modified >= cutoff: + continue + orphans += 1 + if not dry_run: + # Повторная проверка прямо перед удалением: строка могла появиться между запросами. + if conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (key,)).fetchone() is None: + storage.delete(key) + orphans_deleted += 1 + log.info("deleted orphan object %s (no row in PostgreSQL)", key) + + for key, modified, _size in storage.list_objects(): + scanned += 1 + listed.add(key) + page.append((key, modified)) + if len(page) >= 1000: + check(page) + page = [] + if page: + check(page) + + missing = [str(r["id"]) for r in conn.execute( + "SELECT id, object_key FROM attachments WHERE state = 'ready' AND updated_at < %s", (started,)).fetchall() + if r["object_key"] not in listed] + if missing: + log.error("ready attachments without object in S3 (not deleted, needs restore): %s", ", ".join(missing[:20])) + return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted, + "missing_objects": len(missing)} + + +def run_once(grace: int, dry_run: bool) -> dict | None: + started = time.monotonic() + with connect() as conn: + if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]: + log.info("another janitor is running, skip") + return None + try: + found, deleted = reclaim_stale_intents(conn, grace, dry_run) + report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found, + "stale_uploads_deleted": deleted, **sweep_objects(conn, grace, dry_run)} + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,)) + report["seconds"] = round(time.monotonic() - started, 3) + return report + + +def main(): + parser = argparse.ArgumentParser(description="Reclaim abandoned uploads and orphan S3 objects") + parser.add_argument("--once", action="store_true", help="one pass and exit") + parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing") + parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS) + args = parser.parse_args() + if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS: + parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} " + "(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted") + + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + while True: + try: + report = run_once(args.grace_seconds, args.dry_run) + if report is not None: + print(json.dumps(report), flush=True) + failed = False + except (psycopg.Error, storage.StorageUnavailable) as exc: + log.warning("pass failed, will retry: %s", type(exc).__name__) + failed = True + if args.once: + raise SystemExit(1 if failed else 0) + if STOP.wait(settings.JANITOR_INTERVAL_SECONDS): + return + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py new file mode 100644 index 00000000..0cfe9b95 --- /dev/null +++ b/main.py @@ -0,0 +1,583 @@ +import base64 +import binascii +import hashlib +import json +import logging +import re +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path +from typing import Literal +from urllib.parse import quote +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +import attachments +import auth +import db +import settings +import storage +import uploads +from errors import ApiError, bad_request, unauthorized, unavailable + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger") + +DRAINING = threading.Event() + + +class Server(uvicorn.Server): + """Остановка по SIGTERM в два шага. + + 1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS + HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы. + 2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы + (не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM, + получал бы 502 от балансировщика. + Повторный сигнал останавливает сразу. + """ + + def handle_exit(self, sig, frame): + if DRAINING.is_set(): + return super().handle_exit(sig, frame) + DRAINING.set() + log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s", + sig, settings.SHUTDOWN_DRAIN_SECONDS) + threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings.check() + # Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему. + await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS) + # Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3. + Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True) + db.pool.open() + log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s", + settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, settings.SESSION_TTL_SECONDS, + settings.S3_BUCKET) + yield + db.pool.close() + auth.close() + log.info("Stopped: connections closed, data stays in PostgreSQL and Redis") + + +app = FastAPI(lifespan=lifespan) + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH) + + @field_validator("display_name", "password") + @classmethod + def validate_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("value cannot be blank") + return value + + +class User(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(BaseModel): + token: str + token_type: str + expires_at: datetime + user: User + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title cannot be empty") + return value + + +class Chat(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class ChatPage(BaseModel): + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + # С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("attachment_ids must be unique") + if not self.attachment_ids and not self.text.strip(): + raise ValueError("text cannot be empty without attachments") + return self + + +class Message(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: list[UUID] + + +class Variant(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["ready"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + +class MessagePage(BaseModel): + items: list[Message] + next_cursor: str | None + + +def error_response( + request: Request, + code: str, + message: str, + status_code: int, + headers: dict | None = None, +): + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers={"X-Request-Id": request_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + return error_response( + request, + "invalid_request", + "Invalid request data", + 400, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Not found", 404) + return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def general_exception_handler( + request: Request, + exc: Exception, +): + return error_response( + request, + "internal_error", + "Internal server error", + 500, + ) + +PUBLIC_ENDPOINTS = { + ("POST", "/api/v1/users"), + ("POST", "/api/v1/auth/sessions"), +} + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + path = request.url.path + + try: + if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS: + request.state.session = await run_in_threadpool( + auth.check_bearer, request.headers.get("Authorization") + ) + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + except Exception: + log.exception("Unhandled error, request_id=%s", request_id) + response = error_response(request, "internal_error", "Internal server error", 500) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила + # Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки. + level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO + log.log( + level, + "%s %s -> %s (%.1f ms) request_id=%s", + request.method, path, response.status_code, + (time.perf_counter() - started) * 1000, request_id, + ) + return response + + +def current_user_id(request: Request) -> UUID: + session = getattr(request.state, "session", None) + if session is None: + raise unauthorized("Bearer token is required") + return session.user_id + + +def encode_cursor( + created_at: datetime, + object_id: UUID, + *, + chat_id: UUID | None = None, + user_id: UUID | None = None, +) -> str: + data = { + "created_at": created_at.isoformat(), + "id": str(object_id), + } + + if chat_id is not None: + data["chat_id"] = str(chat_id) + + if user_id is not None: + data["user_id"] = str(user_id) + + raw = json.dumps(data).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode_cursor(cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + data = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(data["created_at"]) + + if created_at.tzinfo is None: + raise ValueError("Invalid cursor") + + if not isinstance(data["id"], str): + raise ValueError("Invalid cursor") + + object_id = UUID(data["id"]) + + chat_id = UUID(data["chat_id"]) if "chat_id" in data else None + user_id = UUID(data["user_id"]) if "user_id" in data else None + + return created_at, object_id, chat_id, user_id + + except ( + ValueError, + KeyError, + TypeError, + AttributeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("Invalid cursor") + + +def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): + if cursor is None: + return None + try: + created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) + except ValueError: + raise bad_request("Invalid cursor") from None + if cursor_chat_id != chat_id or cursor_user_id != user_id: + raise bad_request("Invalid cursor") + return created_at, object_id + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + if DRAINING.is_set(): + raise unavailable("Instance is shutting down") + db.ping() + auth.ping_redis() + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=User, status_code=201) +def create_user(user_data: CreateUser): + password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется + return db.create_user(user_data.username, user_data.display_name, password_hash) + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +def create_session( + response: Response, + authorization: str | None = Header(default=None), +): + credentials = auth.parse_basic(authorization) + if credentials is None: + raise auth.login_failed() + username, password = credentials + + found = None + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: + found = db.get_user_with_password(username) + if not auth.password_matches(found["password_hash"] if found else None, password): + raise auth.login_failed() + + token, expires_at = auth.create_session(found["id"]) + response.headers["Cache-Control"] = "no-store" + user = {key: found[key] for key in ("id", "username", "display_name", "created_at")} + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + current_user_id(request) + auth.delete_session(request.state.session) + return Response(status_code=204) + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(request: Request): + user = db.get_user(current_user_id(request)) + if user is None: + raise unauthorized("Session is invalid or expired") + return user + + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, chat_data: CreateChat): + user_id = current_user_id(request) + + if user_id not in chat_data.member_ids: + raise bad_request("Creator must be a chat member") + + if len(chat_data.member_ids) != len(set(chat_data.member_ids)): + raise bad_request("Duplicate member IDs") + return db.create_chat(chat_data.title, chat_data.member_ids) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, user_id=user_id) + + rows = db.list_chats(user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id) + + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID): + return db.get_chat(chat_id, current_user_id(request)) + + +def request_fingerprint(data: BaseModel) -> bytes: + """SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют. + + Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []} + равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними. + """ + body = data.model_dump(mode="json") + if not body.get("attachment_ids"): + body.pop("attachment_ids", None) + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + response_model=Message, + status_code=201, +) +def create_message( + request: Request, + message_data: CreateMessage, + chat_id: UUID, + idempotency_key: str | None = Header(default=None), +): + # С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику) + # вернёт то же сообщение, а не создаст второе. См. db.add_message. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -") + return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids, + idempotency_key, request_fingerprint(message_data)) + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +def list_messages( + request: Request, + chat_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, chat_id=chat_id) + + rows = db.list_messages(chat_id, user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id) + + return MessagePage(items=items, next_cursor=next_cursor) + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=Attachment, + status_code=201, +) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: str | None = Query(default=None), +): + """Синхронная загрузка PNG/JPEG сырыми байтами. 201 — только когда байты в S3 и строка ready в БД. + + Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415) + -> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413), + проверка содержимого (415/413) и запись (attachments.store). + """ + user_id = current_user_id(request) + try: + await run_in_threadpool(db.check_member, chat_id, user_id) + name = uploads.validate_filename(filename) + declared = uploads.declared_type(request.headers.get("content-type")) + uploads.declared_length(request.headers.get("content-length")) + except ApiError: + await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке + raise + upload = await uploads.receive(request) + try: + image = await run_in_threadpool(uploads.inspect, upload, declared) + row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image) + finally: + upload.file.close() # временный файл удаляется сразу; оригинал остаётся только в S3 + return attachments.public(row) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public(db.get_attachment(attachment_id, current_user_id(request))) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + request: Request, + attachment_id: UUID, + variant: Literal["original"] = Query(default="original"), +): + """Байты оригинала через API (без redirect): сначала проверка участника чата, потом S3.""" + row = db.get_attachment(attachment_id, current_user_id(request)) + try: + stream = storage.open_read(row["object_key"]) + except storage.ObjectMissing: + log.error("Object of ready attachment %s is missing in S3", row["id"]) + raise unavailable("File is temporarily unavailable") from None + except storage.StorageUnavailable: + raise unavailable("File storage is temporarily unavailable") from None + headers = { + "Content-Length": str(row["size_bytes"]), + "Content-Disposition": "inline; filename*=UTF-8''" + quote(row["filename"], safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + row["sha256"] + '"', + } + return StreamingResponse(storage.iter_chunks(stream), media_type=row["content_type"], headers=headers) + + +if __name__ == "__main__": + Server(uvicorn.Config( + app, + host=settings.APP_HOST, + port=settings.APP_PORT, + timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS, + access_log=False, + )).run() diff --git a/migrate.py b/migrate.py new file mode 100644 index 00000000..e95f414b --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +import logging + +import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + +if __name__ == "__main__": + db.wait_and_migrate() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..8b981d08 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,33 @@ + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 00000000..73f0c849 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 00000000..5770e1d7 --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,11 @@ +ALTER TABLE messages ADD COLUMN idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); + +ALTER TABLE messages ADD COLUMN request_hash bytea + CHECK (octet_length(request_hash) = 32); + +ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together + CHECK ((idempotency_key IS NULL) = (request_hash IS NULL)); + +ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key + UNIQUE (chat_id, sender_id, idempotency_key); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..35697ddb --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,48 @@ +-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением. +-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать. + +-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата. +ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + -- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит. + object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'), + -- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна; + -- ready: объект в S3 сохранён, можно читать и прикреплять. + state text NOT NULL CHECK (state IN ('uploading', 'ready')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (width::bigint * height <= 40000000), + UNIQUE (id, chat_id), + -- владелец на момент загрузки был участником именно этого чата + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); + +-- janitor ищет старые незавершённые загрузки +CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + -- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id); + +-- С lab4 текст может быть пустым, если есть вложение (проверяет API). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..eb8970d2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +fastapi==0.141.1 +uvicorn==0.54.0 +pydantic==2.13.5 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==6.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +pycparser==3.1 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +boto3==1.43.116 +botocore==1.43.116 +s3transfer==0.19.3 +jmespath==1.1.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +urllib3==2.8.0 +pillow==12.3.0 diff --git a/s3_init.py b/s3_init.py new file mode 100644 index 00000000..99489bc7 --- /dev/null +++ b/s3_init.py @@ -0,0 +1,103 @@ +"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init. + +Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает. +Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py): + admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init; + messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor. +Значения ключей не печатаются. +""" +import logging +import sys +import time +import urllib.error +import urllib.request +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +logging.basicConfig(level=logging.WARNING) + + +def client(key_id: str, secret: str): + return boto3.client( + "s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION, + aws_access_key_id=key_id, aws_secret_access_key=secret, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2, + read_timeout=10, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", response_checksum_validation="when_required"), + ) + + +def code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "?") + + +def anonymous_status(url: str, method: str = "GET") -> int: + try: + with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply: + return reply.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + bucket = settings.S3_BUCKET + admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"), + settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY) + + deadline = time.monotonic() + 90 + while True: + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if time.monotonic() > deadline: + print(f"s3-init: S3 is not ready: {type(exc).__name__}") + return 1 + time.sleep(1) + + try: + admin.create_bucket(Bucket=bucket) + print(f"s3-init: bucket {bucket} created") + except ClientError as exc: + if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + print(f"s3-init: cannot create bucket: {code(exc)}") + return 1 + print(f"s3-init: bucket {bucket} exists") + + # Ключ API работает со своим bucket ... + probe = "originals/s3-init-probe-" + uuid.uuid4().hex + api.put_object(Bucket=bucket, Key=probe, Body=b"probe") + api.get_object(Bucket=bucket, Key=probe)["Body"].read() + api.list_objects_v2(Bucket=bucket, MaxKeys=1) + api.delete_object(Bucket=bucket, Key=probe) + print("s3-init: api key can put/get/list/delete objects of its bucket") + # ... и не может администрировать хранилище. + intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8] + try: + api.create_bucket(Bucket=intruder) + admin.delete_bucket(Bucket=intruder) + print("s3-init: api key must NOT be able to create buckets") + return 1 + except ClientError as exc: + print(f"s3-init: api key cannot create buckets ({code(exc)})") + + # Приватность: анонимные запросы без подписи. + base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket + statuses = {"list": anonymous_status(base + "?list-type=2"), + "get": anonymous_status(base + "/originals/any"), + "put": anonymous_status(base + "/originals/any", "PUT")} + if any(status not in (401, 403) for status in statuses.values()): + print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}") + return 1 + print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..b69f27fd --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,77 @@ +import json +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + with open(SECRETS / name, "w", encoding="utf-8", newline="") as file: + file.write(value) + (SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore + + +def bucket_name() -> str: + """S3_BUCKET из .env (если задан), иначе значение по умолчанию.""" + env = ROOT / ".env" + if env.exists(): + for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("S3_BUCKET="): + return line.split("=", 1)[1].strip() + return "messenger-attachments" + + +def s3_config(bucket: str) -> str: + """Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket. + + Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects) + с суффиксом : действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет. + Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются. + """ + def read(name): + return (SECRETS / name).read_text(encoding="utf-8").strip() + config = {"identities": [ + {"name": "admin", + "credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ]} + return json.dumps(config, indent=2) + + +def main() -> None: + """Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes).""" + SECRETS.mkdir(exist_ok=True) + created = [] + if not (SECRETS / "postgres_password").exists(): + db_password = secrets.token_hex(24) + redis_password = secrets.token_hex(24) + write("postgres_password", db_password) + write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger") + write("cache_url", f"redis://:{redis_password}@redis:6379/0") + write("redis_auth", f"requirepass {redis_password}\n") + created += ["postgres_password", "database_url", "cache_url", "redis_auth"] + if not (SECRETS / "s3_root_password").exists(): + # admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init + write("s3_root_user", "root-" + secrets.token_hex(6)) + write("s3_root_password", secrets.token_hex(24)) + created += ["s3_root_user", "s3_root_password"] + if not (SECRETS / "s3_secret_access_key").exists(): + # отдельный ключ API с политикой только на свой bucket (создаёт s3-init) + write("s3_access_key_id", "api-" + secrets.token_hex(8)) + write("s3_secret_access_key", secrets.token_hex(24)) + created += ["s3_access_key_id", "s3_secret_access_key"] + if not (SECRETS / "s3_config.json").exists(): + write("s3_config.json", s3_config(bucket_name())) + created.append("s3_config.json") + if created: + print("Created in secrets/: " + ", ".join(created) + " (values are not printed).") + else: + print("secrets/ is complete, nothing changed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..29bb4e7b --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,13 @@ +@echo off +rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then api and janitor, then the balancer. Volumes are kept. +rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +cd /d "%~dp0.." +docker compose restart postgres redis s3 || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 || exit /b 1 +docker compose restart api janitor || exit /b 1 +docker compose restart lb || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps api lb || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..d09ac37f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Restart hook for tests/resilience.py --mode persistence (Linux/macOS). +# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then all api replicas and janitor, then the balancer. +# Volumes are kept (no down -v), nothing is re-seeded. +# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres redis s3 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 +docker compose restart api janitor +docker compose restart lb +docker compose up -d --wait --no-recreate --no-build --no-deps api lb diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..8ffd0fa2 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py. +python "%~dp0stop_one.py" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..cf4d7244 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID. +# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd. +set -eu +exec python3 "$(dirname "$0")/stop_one.py" diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 00000000..5cdcce70 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STOP_TIMEOUT_SECONDS = 8 + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{12,64}", target): + print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr) + return 2 + running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + matches = [container for container in running if container.startswith(target)] + if len(matches) != 1: + print(f"no running api container with id {target}", file=sys.stderr) + return 1 + subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + print(f"stopped api container {matches[0][:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/settings.py b/settings.py new file mode 100644 index 00000000..06930544 --- /dev/null +++ b/settings.py @@ -0,0 +1,86 @@ +import os +import socket +from pathlib import Path + + +def env_str(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def env_int(name: str, default: int) -> int: + value = int(env_str(name, str(default))) + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value + + +def read_secret(file_variable: str, plain_variable: str) -> str: + path = env_str(file_variable, "") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return env_str(plain_variable, "") + + +APP_HOST = env_str("APP_HOST", "0.0.0.0") +APP_PORT = env_int("APP_PORT", 8080) +APP_ENV = env_str("APP_ENV", "lab") +APP_VERSION = env_str("APP_VERSION", "dev") + +INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname() + +SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600) +SESSION_STORE = env_str("SESSION_STORE", "redis") + +DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL") +DB_POOL_MIN = env_int("DB_POOL_MIN", 1) +DB_POOL_MAX = env_int("DB_POOL_MAX", 10) +DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3) +DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3) +DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + +REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL") +REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20) +REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500) + +MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations"))) +SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60) + +# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket. +S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333") +S3_REGION = env_str("S3_REGION", "us-east-1") +S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1) +S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5) +S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2) +S3_POOL_MAX = env_int("S3_POOL_MAX", 20) + +# Вложения: лимиты из контракта и защита памяти/времени. +MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10) +DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2) +UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads") + +# Очистка незавершённых загрузок (janitor.py). +ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600) +JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300) + +SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3) +SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10) + + +def check() -> None: + """Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе.""" + if not DATABASE_URL: + raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE") + if not REDIS_URL: + raise RuntimeError("Redis address is not set: CACHE_URL_FILE") + if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY: + raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE") + if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS: + raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS") + if SESSION_STORE != "redis": + raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution") diff --git a/storage.py b/storage.py new file mode 100644 index 00000000..1c88e16d --- /dev/null +++ b/storage.py @@ -0,0 +1,153 @@ +"""Приватное S3-хранилище (SeaweedFS) для байтов вложений. + +API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket +(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер, +имя файла пользователя в ключ не попадает. +""" +import base64 +import logging +from datetime import datetime +from typing import BinaryIO, Iterator + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +log = logging.getLogger("messenger.storage") + +ORIGINALS_PREFIX = "originals/" +READ_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей.""" + + +class ObjectMissing(Exception): + """Объекта нет в bucket (NoSuchKey / 404).""" + + +def _make_client(connect_timeout: float, read_timeout: float, attempts: int): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT, + region_name=settings.S3_REGION, + aws_access_key_id=settings.S3_ACCESS_KEY_ID or None, + aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, # http://s3:8333// + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую + max_pool_connections=settings.S3_POOL_MAX, + # Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно. + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + +_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS) +# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor. +_quick_client = _make_client(1, 2, 1) + + +def _is_missing(exc: ClientError) -> bool: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("NoSuchKey", "404", "NotFound") or status == 404 + + +def _unavailable(action: str, exc: Exception) -> StorageUnavailable: + detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else "" + log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail) + return StorageUnavailable() + + +def new_key(object_id: str) -> str: + return ORIGINALS_PREFIX + object_id + + +def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + """Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект.""" + body.seek(0) + try: + _client.put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("put", exc) from None + + +def open_read(key: str): + """GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после.""" + try: + return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"] + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing(key) from None + raise _unavailable("get", exc) from None + except BotoCoreError as exc: + raise _unavailable("get", exc) from None + + +def iter_chunks(stream) -> Iterator[bytes]: + """Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание.""" + try: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return + yield chunk + except (BotoCoreError, OSError) as exc: + # Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length). + log.warning("S3 stream interrupted: %s", type(exc).__name__) + raise + finally: + stream.close() + + +def delete(key: str, quick: bool = False) -> None: + """Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа.""" + try: + (_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("delete", exc) from None + + +def exists(key: str) -> bool: + try: + _client.head_object(Bucket=settings.S3_BUCKET, Key=key) + return True + except ClientError as exc: + if _is_missing(exc): + return False + raise _unavailable("head", exc) from None + except BotoCoreError as exc: + raise _unavailable("head", exc) from None + + +def list_objects(prefix: str = ORIGINALS_PREFIX) -> Iterator[tuple[str, datetime, int]]: + """Все объекты под prefix: (key, last_modified, size). Постранично, по 1000.""" + paginator = _client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"], item["Size"] + except (BotoCoreError, ClientError) as exc: + raise _unavailable("list", exc) from None + + +def ping() -> None: + try: + _client.head_bucket(Bucket=settings.S3_BUCKET) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("head_bucket", exc) from None diff --git a/student-tests/check_stack.py b/student-tests/check_stack.py new file mode 100644 index 00000000..3c91f0d3 --- /dev/null +++ b/student-tests/check_stack.py @@ -0,0 +1,78 @@ + +import json +import platform +import sys + +from common import call, check, compose_output, docker, new_user, seen_instances + + +def main(): + info = json.loads(docker("info", "--format", "{{json .}}")) + print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"client Python {platform.python_version()}") + print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, " + f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, " + f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}") + + print() + print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip()) + for one_shot in ("migrate", "s3-init"): + status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip() + check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0") + + config = json.loads(compose_output("config", "--format", "json")) + services = config["services"] + containers = compose_output("ps", "-q").split() + rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()] + print() + print("container image cpus memory host ports networks") + api_count = 0 + for row in sorted(rows, key=lambda r: r["Name"]): + labels = row["Config"]["Labels"] + service = labels.get("com.docker.compose.service") + limits = row["HostConfig"] + cpus = limits.get("NanoCpus", 0) / 1e9 + memory = limits.get("Memory", 0) // 2**20 + ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values() + for b in (bindings or [])}) + networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"]) + print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB " + f"{','.join(ports) or '-':<23} {','.join(networks)}") + check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit") + if service == "api": + api_count += 1 + check(not ports, "api must not publish ports: everything goes through the balancer") + if service == "lb": + check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only") + if service in ("postgres", "redis", "s3", "janitor"): + check(not ports and networks == ["data"], f"{service} must be only in the internal data network") + check(api_count >= 2, "need at least 2 api replicas") + check(all("container_name" not in s for s in services.values()), "container_name prevents scaling") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis") + + print() + for name, service in sorted(services.items()): + secrets = sorted(s["source"] for s in service.get("secrets", [])) + print(f"secrets of {name}: {secrets or '-'}") + check(not services["lb"].get("secrets"), "lb gets no secrets") + check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url") + for name in ("api", "janitor"): + names = {s["source"] for s in services[name]["secrets"]} + check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys") + check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key") + + user = new_user("stack") + served = seen_instances(user["token"]) + print() + print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}") + check(len(served) >= 2, "requests are not balanced") + health = call("GET", "/health/ready", 200) + print(f"/health/ready -> {health.status} from {health.instance}") + print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every " + f"container, postgres/redis/s3/janitor only in the internal data network, " + f"S3 admin keys only in s3 and s3-init") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 00000000..60dbdac5 --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,293 @@ + +import json +import os +import platform +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PY = sys.executable + +STEPS = [ + ("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]), + ("stack", "stand layout: replicas, ports, limits, networks, secrets, S3", [PY, "student-tests/check_stack.py"]), + ("smoke", "public smoke via the balancer (make test), with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]), + ("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids", + [PY, "student-tests/test_attachments.py"]), + ("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap", + [PY, "student-tests/test_s3_security.py"]), + ("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage", + [PY, "student-tests/test_consistency.py"]), + ("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas", + [PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]), + ("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]), + ("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]), + ("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]), + ("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]), + ("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]), + ("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]), + ("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]), + ("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)", + [PY, "student-tests/run_failover.py"]), + ("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL, + "--duration", "30", "--concurrency", "4", + "--output", "evidence/load.json"]), + ("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]), + ("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)", + [PY, "student-tests/run_persistence.py"]), +] + + +def git(*args): + try: + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "" + + +def commit(): + head = git("rev-parse", "--short", "HEAD") or "unknown" + changed = [line for line in git("status", "--porcelain").splitlines() + if not line[3:].startswith(("evidence/", "REPORT.md"))] + return head + (" + uncommitted changes" if changed else "") + + +def run_step(name, description, command): + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL) + print(f"=== {name}: {description}", flush=True) + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + seconds = time.monotonic() - started + output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n") + shown = " ".join(Path(part).name if part == PY else part for part in command) + header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n" + f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n") + (EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8") + tail = [line for line in output.splitlines() if line.strip()][-3:] + for line in tail: + print(" " + line) + print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True) + return result.returncode == 0, seconds + + +def load_json(name): + path = EVIDENCE / name + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + +def measurements(): + """Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md).""" + lines = [] + try: + info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True, + text=True, check=True).stdout) + docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM" + except (OSError, ValueError, subprocess.CalledProcessError): + docker_vm = "unknown" + lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона " + f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB " + f"каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, " + f"janitor 0,25 CPU / 192 MiB.") + lines.append("") + + failover = EVIDENCE / "failover.txt" + results = [] + if failover.exists(): + text = failover.read_text(encoding="utf-8") + stopped = re.findall(r"the hook stopped (\w+)", text) + results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)] + if results: + lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):") + lines.append("") + lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток " + "(повторены с тем же ключом) | Первая запись после hook, с |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for number, (result, container) in enumerate(zip(results, stopped), start=1): + lines.append(f"| {number} | {container} | {result['observed_instances_before']} | " + f"{result['successful_writes']} | {result['failed_attempts']} | " + f"{result['first_recovery_seconds']} |") + lines.append("") + + memory = load_json("upload-memory.json") + if memory: + lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики " + f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). " + f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:") + lines.append("") + lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | " + "Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for run in memory["runs"]: + for n, r in enumerate(run["replicas"]): + head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | " + f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |" + cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-" + lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | " + f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |") + naive = memory["single_file_extra_rss_mib"] + lines.append("") + lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в " + f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка " + f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в " + f"{memory['api_limit_mib']} MiB.") + lines.append("") + + metrics = load_json("failover-metrics.json") + if metrics: + lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи " + "без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):") + lines.append("") + lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок " + "(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | " + "Вернулась в балансировку после docker start, с |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for r in metrics["runs"]: + how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)" + codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-" + lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | " + f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | " + f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |") + lines.append("") + lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика " + "после остановки не ответила ни разу.") + lines.append("") + + rolling = load_json("rolling-update.json") + if rolling: + during = rolling["during"] + codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-" + lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> " + f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за " + f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, " + f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); " + f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено " + f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.") + lines.append("") + + load = load_json("load.json") + if load: + latency = load["successful_latency_ms"] + lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, " + f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных " + f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных " + f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.") + lines.append("") + + persistence = EVIDENCE / "persistence.txt" + restart_seconds = None + if persistence.exists(): + found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8")) + restart_seconds = found.group(1) if found else None + + impacts = [] + if metrics: + for mode, what, expected in ( + ("stop", "`docker stop` реплики (SIGTERM) под нагрузкой", + "ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"), + ("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой", + "единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")): + runs = [r for r in metrics["runs"] if r["mode"] == mode] + if runs: + impacts.append( + f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); " + f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | " + f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; " + f"потерь 0, дублей 0; реплика вернулась через " + f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |") + if failover.exists() and results: + impacts.append( + f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, " + f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: " + f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через " + f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |") + if rolling: + impacts.append( + f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | " + f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |") + if restart_seconds: + impacts.append( + f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и " + f"прежний токен живы | " + f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |") + consistency = EVIDENCE / "consistency.txt" + if consistency.exists(): + text = consistency.read_text(encoding="utf-8") + timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text) + if len(timings) >= 2: + impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, " + f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | " + f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |") + impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за " + f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |") + if "PASS: S3 written but DB not committed" in text: + impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | " + "вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | " + "GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | " + "мусора не осталось; повторный проход ничего не делает |") + if "PASS: an interrupted upload leaves no row" in text: + impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | " + "строк и объектов не прибавилось, временных файлов 0 | мгновенно |") + if impacts: + lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |") + lines.append("| --- | --- | --- | --- |") + lines.extend(impacts) + lines.append("") + return "\n".join(lines).strip() + + +def update_report(block): + report = ROOT / "REPORT.md" + text = report.read_text(encoding="utf-8") + begin, end = "", "" + if begin not in text or end not in text: + print("REPORT.md has no measurement markers, only evidence/summary.md was written") + return + head, rest = text.split(begin, 1) + _, tail = rest.split(end, 1) + report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8") + print("REPORT.md: measurement tables updated") + + +def main(): + args = sys.argv[1:] + names = [name for name, _, _ in STEPS] + if args[:1] == ["--from"]: + if len(args) < 2 or args[1] not in names: + raise SystemExit("usage: --from ; steps: " + ", ".join(names)) + selected = names[names.index(args[1]):] + else: + unknown = [a for a in args if a not in names] + if unknown: + raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names)) + selected = args or names + EVIDENCE.mkdir(exist_ok=True) + results = [] + for name, description, command in STEPS: + if name not in selected: + continue + ok, seconds = run_step(name, description, command) + results.append((name, description, ok, seconds)) + if not ok: + print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.") + print(f"After fixing: python student-tests/collect_evidence.py --from {name}") + break + block = measurements() + summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |", + "| --- | --- | --- | --- | --- |"] + summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results] + (EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8") + print("evidence/summary.md written") + passed = all(ok for _, _, ok, _ in results) + if passed: + update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 00000000..7fed6ffb --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,324 @@ + +import base64 +import csv +import hashlib +import io +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +PASSWORD = "Test-password-123" + + +class Response: + def __init__(self, status, headers, data): + self.status = status + self.headers = {name.lower(): value for name, value in headers.items()} + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except ValueError: + self.json = None + + @property + def instance(self): + """X-Instance-Id: какая реплика API ответила (ID её контейнера).""" + return self.headers.get("x-instance-id") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30): + """HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError. + + Заодно проверяет то, что контракт требует от любого ответа: + X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401. + """ + all_headers = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + if data is not None: + all_headers.setdefault("Content-Type", "application/json") + if token is not None: + all_headers["Authorization"] = "Bearer " + token + + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + response = Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + response = Response(error.code, error.headers, error.read()) + + expected = expected if isinstance(expected, tuple) else (expected,) + check(response.status in expected, + f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}") + request_id = response.headers.get("x-request-id") + check(request_id, f"{method} {path}: no X-Request-Id header") + check(response.instance, f"{method} {path}: no X-Instance-Id header") + if response.status >= 400: + error = (response.json or {}).get("error") or {} + check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format") + check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header") + if response.status == 401: + check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate") + return response + + +def try_call(method, path, *, body=None, token=None, headers=None, timeout=2): + """Запрос без проверок и без исключений — для нагрузки во время отказа. + + Возвращает Response или None, если соединение оборвалось / вышел timeout. + """ + all_headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + all_headers["Content-Type"] = "application/json" + if token is not None: + all_headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + return Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + try: + return Response(error.code, error.headers, error.read()) + except OSError: + return None + except (urllib.error.URLError, OSError): + return None + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8.""" + return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + + +def login(username, password, expected=201): + return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)}) + + +def new_user(prefix="user", password=PASSWORD): + """Регистрирует пользователя и входит. Возвращает dict: id, username, password, token.""" + username = f"{prefix}_{uuid.uuid4().hex[:10]}" + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": prefix.title(), "password": password}).json + session = login(username, password).json + return {**user, "password": password, "token": session["token"]} + + +def new_chat(owner, *others, title="Test chat"): + member_ids = [owner["id"]] + [user["id"] for user in others] + return call("POST", "/api/v1/chats", 201, token=owner["token"], + body={"title": title, "member_ids": member_ids}).json + + +def new_key(): + return uuid.uuid4().hex + + +def send(user, chat_id, text, expected=201, key=None): + """POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body={"text": text}, headers={"Idempotency-Key": key or new_key()}) + + +def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None): + """POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся.""" + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body=body, headers={"Idempotency-Key": key or new_key()}) + + +def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=201, timeout=60): + """POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра.""" + query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="") + headers = {"Content-Type": content_type} if content_type is not None else {} + return call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data, + token=user["token"] if user else None, headers=headers, timeout=timeout) + + +def read_history(user, chat_id, limit=100): + """Вся история чата по страницам. Возвращает (сообщения, число_страниц).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "") + page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json + items += page["items"] + pages += 1 + cursor = page["next_cursor"] + if cursor is None: + return items, pages + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while True: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply: + if reply.status == 200: + return + except (urllib.error.URLError, OSError): + pass + check(time.monotonic() < deadline, f"API is not ready after {seconds} s") + time.sleep(0.5) + + +def compose(*args, env=None): + """docker compose в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам.""" + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def compose_output(*args, env=None): + """docker compose и его вывод (для logs, ps).""" + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env) + return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace") + + +def docker(*args, check_exit=True): + """docker , возвращает stdout.""" + result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit) + return result.stdout.decode("utf-8", "replace").strip() + + +def api_containers(include_stopped=False): + """Полные ID контейнеров сервиса api (по умолчанию только запущенные).""" + args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"] + return sorted(compose_output(*args).split()) + + +def health(container): + """Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''.""" + return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container, + check_exit=False) + + +def lb_servers(): + """Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}.""" + with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply: + rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace")))) + header = [name.lstrip("# ") for name in rows[0]] + servers = {} + for row in rows[1:]: + item = dict(zip(header, row)) + if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"): + servers[item["svname"]] = item.get("status", "") + return servers + + +def lb_up_count(): + """Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP).""" + return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP")) + + +def seen_instances(token, requests=20): + """Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me.""" + return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)} + + +def wait_balanced(replicas, seconds=60): + """Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке.""" + deadline = time.monotonic() + seconds + while True: + running = api_containers() + healthy = [container for container in running if health(container) == "healthy"] + try: + up = lb_up_count() + except OSError: + up = 0 + if len(healthy) >= replicas and up >= replicas: + return running + check(time.monotonic() < deadline, + f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}") + time.sleep(0.5) + + +def redis_cli(*args): + """redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод.""" + command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args) + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command], + cwd=ROOT, capture_output=True, check=True) + return result.stdout.decode("utf-8", "replace").strip() + + +def session_key(token): + """Ключ сессии в Redis, как его считает auth.py: session:.""" + return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def psql(sql): + """Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки). + + Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя. + """ + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, check=False, + ) + output = result.stdout.decode("utf-8", "replace").strip() + error = result.stderr.decode("utf-8", "replace").strip() + return result.returncode, output, error + + +def app_python(code, *args, service="api", env=None): + """Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API). + + Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON). + """ + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return output + + +def janitor(*args, env=None): + """Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт.""" + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += ["janitor", "python", "janitor.py", "--once", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return json.loads(output.splitlines()[-1]) + + +def sql_value(query): + """Одно значение из PostgreSQL (psql -A -t).""" + code, output, error = psql(query) + check(code == 0, f"psql failed: {error[-300:]}") + return output + + +def run_tests(tests): + """Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал.""" + failed = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста + failed += 1 + print(f"FAIL: {test.__name__}: {exc}") + print(f"{len(tests) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/demo_instances.py b/student-tests/demo_instances.py new file mode 100644 index 00000000..8f97f1c3 --- /dev/null +++ b/student-tests/demo_instances.py @@ -0,0 +1,16 @@ + +from common import call, compose_output, new_user + + +def main(): + user = new_user("demo") + print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:") + for number in range(1, 9): + reply = call("GET", "/api/v1/users/me", 200, token=user["token"]) + print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}") + print() + print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 00000000..81416de6 --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,28 @@ + +import subprocess + +from common import ROOT + +OUTPUT = ROOT / "evidence" / "explain-history.txt" + + +def main(): + sql = (ROOT / "student-tests" / "explain.sql").read_bytes() + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + cwd=ROOT, input=sql, capture_output=True, check=False, + ) + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_bytes(result.stdout + result.stderr) + if result.returncode != 0: + print(result.stderr.decode("utf-8", "replace")) + raise SystemExit("FAIL: psql returned an error, see output above") + for line in result.stdout.decode("utf-8", "replace").splitlines(): + if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line: + print(line) + print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 00000000..01cd9e70 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,76 @@ + +\pset footer off +BEGIN; + +CREATE TEMP TABLE seed_chats AS + SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size + FROM generate_series(0, 50) AS n; +CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; + +INSERT INTO users (id, username, display_name, password_hash) + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; +INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), + timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' + FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; +ANALYZE messages; + +SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset +SELECT created_at AS cursor_time, id AS cursor_id FROM messages + WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== Data volume' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat +FROM messages; + +\echo '' +\echo '=== History index' +SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history'; + +\echo '' +\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY m.created_at, m.id LIMIT 51; + +\echo '' +\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id OFFSET 10000 LIMIT 51; + +\echo '' +\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)' +DROP INDEX messages_history; +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; +\echo '' +\echo '=== ROLLBACK: test data removed, index is back' +SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%'; diff --git a/student-tests/failover_metrics.py b/student-tests/failover_metrics.py new file mode 100644 index 00000000..19421a15 --- /dev/null +++ b/student-tests/failover_metrics.py @@ -0,0 +1,129 @@ +import json +import platform +import sys +import threading +import time + +from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced +from loadgen import Load, summary + +WARMUP_SECONDS = 3 +AFTER_SECONDS = 10 +OUTPUT = ROOT / "evidence" / "failover-metrics.json" + + +class LbWatch: + """Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP).""" + + def __init__(self): + self.points, self.stopping = [], threading.Event() + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stopping.is_set(): + try: + self.points.append((time.monotonic(), lb_up_count())) + except OSError: + pass + time.sleep(0.1) + + def start(self): + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join(timeout=3) + + def first_drop(self, after, replicas): + return next((t for t, up in self.points if t >= after and up < replicas), None) + + +def measure(container, mode, replicas): + wait_balanced(replicas) + load, watch = Load(), LbWatch() + watch.start() + load.start() + time.sleep(WARMUP_SECONDS) + t0 = time.monotonic() + if mode == "stop": + docker("stop", "--time", "8", container) + else: + docker("kill", container) + command_seconds = time.monotonic() - t0 + time.sleep(max(0.0, AFTER_SECONDS - command_seconds)) + end = t0 + AFTER_SECONDS + load.stop() + watch.stop() + + errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]] + errors_end = max((e[0] + e[5] for e in errors), default=None) + drop = watch.first_drop(t0, replicas) + served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]] + target_hits = sum(1 for instance in served_after if instance and container.startswith(instance)) + integrity = load.verify() + + started = time.monotonic() + docker("start", container) + wait_balanced(replicas) + rejoin_seconds = time.monotonic() - started + + return { + "replica": container[:12], + "mode": mode, + "command_seconds": round(command_seconds, 2), + "lb_excluded_after_seconds": round(drop - t0, 2) if drop else None, + "errors": len(errors), + "error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"], + "errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0, + "before": summary(load, t0 - WARMUP_SECONDS, t0), + "after": summary(load, t0, end), + "stopped_replica_answers_after_stop": target_hits, + "integrity": integrity, + "rejoin_seconds": round(rejoin_seconds, 2), + } + + +def main(): + modes = sys.argv[1:] or ["stop", "kill"] + check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]") + containers = api_containers() + replicas = len(containers) + check(replicas >= 2, "need at least 2 running api replicas") + print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, " + f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and " + f"{AFTER_SECONDS} s after the failure") + runs = [] + for container in containers: + for mode in modes: + print(f"--- {mode} {container[:12]} ...") + run = measure(container, mode, replicas) + runs.append(run) + print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after " + f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, " + f"stopped after {run['errors_stopped_after_seconds']} s") + print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; " + f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; " + f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}") + print(f" confirmed {run['integrity']['confirmed_writes']}, in history " + f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; " + f"back in the balancer {run['rejoin_seconds']} s after docker start") + check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered") + check(run["after"]["write_rps"] > 0, "no successful writes after the failure") + print() + print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s") + for r in runs: + print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} " + f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} " + f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS, + "after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()}, + "runs": runs, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; " + f"saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/progressive.jpg b/student-tests/fixtures/progressive.jpg new file mode 100644 index 0000000000000000000000000000000000000000..38faa2bd4a2bb00b572233684dd5474d46572d38 GIT binary patch literal 1081 zcmex=^lOiET&UP@Y7ModgWM?qOlT~kX_QeM|USHnP6LsJ7}2qQZ?I~NC+Fc+7w zhLo6;2Fc+6AqGJXh5&{NW=16jCP7AKLB{__7$kv?WCpnw4wxC3SlHNsQWA_{0GD83 zU}0ipXa0YSfrpuqfk}W_0O-e0Us7g@q%7gR_iLh?m&0PUC4!SCXq;RjxhlxgB!%5l zB;^Y4y)P5pd>t06DhN&v(m3fMxhj9ZT2I4sr=BT|yYB~Rob&`LVloP1l{8g0_4@wg z;slu|fs=#Gt1kgXlzOH#fHkY6t=#u#mJ#!uE~6l}_q85C?}^ISER2f9~O z0L335cMAd?%K;Q+WM%509st@UCI}20q>y1?U}9hdTLuhFdqzD7I4%wW$=wjJ@hAk` zcmx4&%0bQs8iisK%-KvJlU88?Kusb7=z0pEUS+dq62l%C5cdHC!(Oo`S>l)>t%&2W z6vDs^^q0V|se8Fk9r(1>j{TMMuhu{Ow;I1Ktz-Vh1QNe>2U zW@aX!`+*(+s;&jfGq4D<3Mm>ovIz$!vMUve7&T5@$f4}C@t|nX#SbdRNkvVZTw>x9 zl2WQ_>Kd9_CZ=ZQ7M51dF0O9w9-dyoA)#U65s^{JDXD4c8JStdC8cHM6_r)ZEv;?s z9i3g1CQq3GGAU*RJ2V zdF$b$$4{OPfBE|D`;VW$K>lK6V1{@LNJ2b@<}X2@znFm0!om*n7b88f2KE_o9%~}Y zXK;@p{B?_ghnW!=dCYp^Nk zI`iL#Ki&>f`#Y>_;hTV?BFa#mKfu=BySiva)fccMR;=xKyLZJ}+l8;3MHfC61R6dA zsN_%Cy{lsJIJc*Fss0-!LA1S;hP4?4;WTe zNi6EY^g|NJ;Qc@!feePYdTl7gV2F?4&Tusb`z-gZYYxzbu2CzhCLGPE06H8TNU&hp z7a3yT2M$D#1O7sTygm!&S4fcV2U`gY#9&*Xv# bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes: + """PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой). + + pad_to — дополнить файл служебным чанком prVt до точного размера в байтах. + idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое). + """ + channels = 1 if gray else 3 + header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace) + if idat is None: + compressor = zlib.compressobj(6) + parts = [] + # Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход. + passes = [(width, height)] if not interlace else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + for pass_width, pass_height in passes: + if pass_width <= 0 or pass_height <= 0: + continue + row = b"\x00" + bytes(pass_width * channels) + for _ in range(pass_height): + parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row)) + parts.append(compressor.flush()) + idat = b"".join(parts) + body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat) + if pad_to is not None: + need = pad_to - len(body) - 12 - 12 # чанк-заполнитель + IEND + assert need >= 0, "image is already larger than pad_to" + body += chunk(b"prVt", bytes(need)) + return body + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями).""" + side = int(((target_bytes - 4096) / 3) ** 0.5) + return png(side, side, noise=True) + + +def sample_png() -> bytes: + return (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (FIXTURES / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (FIXTURES / "progressive.jpg").read_bytes() + + +def flip_byte(data: bytes, position: int) -> bytes: + return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:] + + +GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 00000000..998548cf --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,102 @@ +import math +import threading +import time + +from common import check, new_chat, new_key, new_user, read_history, try_call + + +class Load: + def __init__(self, writers=4, timeout=2): + self.alice, self.bob = new_user("load_a"), new_user("load_b") + self.chat = new_chat(self.alice, self.bob, title="load") + self.writers, self.timeout = writers, timeout + self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с) + self.confirmed = [] # сообщения, на которые пришёл 201 + self.lock = threading.Lock() + self.stopping = threading.Event() + self.threads = [] + + def _record(self, kind, started, reply, ok): + code = reply.status if reply is not None else "conn" + instance = reply.instance if reply is not None else None + with self.lock: + self.events.append((started, kind, ok, code, instance, time.monotonic() - started)) + + def _writer(self, number): + path = f"/api/v1/chats/{self.chat['id']}/messages" + sent = 0 + key, text = new_key(), f"load-{number}-{sent}" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("POST", path, body={"text": text}, token=self.alice["token"], + headers={"Idempotency-Key": key}, timeout=self.timeout) + ok = reply is not None and reply.status == 201 + self._record("write", started, reply, ok) + if ok: + with self.lock: + self.confirmed.append(reply.json) + sent += 1 + key, text = new_key(), f"load-{number}-{sent}" + else: + time.sleep(0.05) # и повторить тот же text с тем же ключом + + def _reader(self): + path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout) + self._record("read", started, reply, reply is not None and reply.status == 200) + time.sleep(0.05) + + def start(self): + self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)] + self.threads.append(threading.Thread(target=self._reader, daemon=True)) + for thread in self.threads: + thread.start() + + def stop(self): + self.stopping.set() + for thread in self.threads: + thread.join(timeout=self.timeout + 5) + + def window(self, start, end, kind=None): + with self.lock: + return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)] + + def verify(self): + """Все подтверждённые сообщения есть в истории, id и тексты не повторяются.""" + history, _ = read_history(self.bob, self.chat["id"]) + ids = [message["id"] for message in history] + texts = [message["text"] for message in history] + by_id = {message["id"]: message for message in history} + check(len(ids) == len(set(ids)), "duplicated message ids in history") + check(len(texts) == len(set(texts)), "a retried request created a duplicate message") + lost = [m for m in self.confirmed if by_id.get(m["id"]) != m] + check(not lost, f"{len(lost)} confirmed messages are lost or changed") + return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0} + + +def summary(load, start, end): + """Успешные RPS, ошибки и задержки на отрезке [start, end).""" + events = load.window(start, end) + writes = [e for e in events if e[1] == "write"] + ok_writes = sorted(e[5] for e in writes if e[2]) + errors = [e for e in events if not e[2]] + seconds = max(end - start, 1e-6) + + def percentile(p): + return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None + + codes = {} + for e in errors: + codes[str(e[3])] = codes.get(str(e[3]), 0) + 1 + return { + "seconds": round(seconds, 2), + "write_rps": round(len(ok_writes) / seconds, 1), + "reads": len([e for e in events if e[1] == "read"]), + "errors": len(errors), + "error_codes": codes, + "write_p50_ms": percentile(0.50), + "write_p95_ms": percentile(0.95), + "write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None, + } diff --git a/student-tests/redis_info.py b/student-tests/redis_info.py new file mode 100644 index 00000000..6535eae2 --- /dev/null +++ b/student-tests/redis_info.py @@ -0,0 +1,29 @@ +import subprocess + +from common import ROOT, login, new_user, redis_cli, session_key + + +def without_password(): + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"], + cwd=ROOT, capture_output=True, check=False) + return (result.stdout + result.stderr).decode("utf-8", "replace").strip() + + +def main(): + user = new_user("info") + token = login(user["username"], user["password"]).json["token"] + key = session_key(token) + print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}") + print(f"TTL of the key, seconds: {redis_cli('ttl', key)}") + keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines() + print(f"sessions in Redis: {len(keys)}") + for name in ("appendonly", "appendfsync", "maxmemory-policy"): + print("config", " = ".join(redis_cli("config", "get", name).split())) + for line in redis_cli("info", "persistence").splitlines(): + if line.startswith(("aof_enabled", "aof_last_write_status")): + print(line) + print(f"without password: {without_password()}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 00000000..f2f3ffbb --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,88 @@ +import json +import os +import time + +from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced +from loadgen import Load, summary + +OUTPUT = ROOT / "evidence" / "rolling-update.json" + + +def env_of(container): + lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +def wait_new_replica(container, up_needed, seconds=90): + deadline = time.monotonic() + seconds + while health(container) != "healthy" or lb_up_count() < up_needed: + check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") + time.sleep(0.2) + + +def main(): + old = api_containers() + replicas = len(old) + check(replicas >= 2, "need at least 2 running api replicas") + wait_balanced(replicas) + old_version = env_of(old[0]).get("APP_VERSION") + version = time.strftime("rolling-%Y%m%d-%H%M%S") + env = dict(os.environ, APP_VERSION=version) + + print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") + started = time.monotonic() + compose("build", "api", "migrate", env=env) + print(f"image built in {time.monotonic() - started:.1f} s") + migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) + print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] + .split("messenger.db: ")[-1]) + + load = Load() + load.start() + time.sleep(3) + t0 = time.monotonic() + steps = [] + for container in old: + step_start = time.monotonic() + before = set(api_containers()) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) + new = sorted(set(api_containers()) - before) + check(len(new) == 1, f"expected one new container, got {len(new)}") + wait_new_replica(new[0], replicas + 1) + ready = time.monotonic() - step_start + docker("stop", "--time", "15", container) + docker("rm", container) + steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), + "step_seconds": round(time.monotonic() - step_start, 2)}) + print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " + f"step {steps[-1]['step_seconds']} s") + t1 = time.monotonic() + time.sleep(3) + load.stop() + + final = api_containers() + versions = sorted({env_of(c).get("APP_VERSION") for c in final}) + integrity = load.verify() + during = summary(load, t0, t1) + before_stats = summary(load, t0 - 3, t0) + print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") + print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " + f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " + f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") + print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " + f"lost 0, duplicates 0") + check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") + check(versions == [version], "not every replica runs the new version") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "from_version": old_version, "to_version": version, "replicas": replicas, + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, + "steps": steps, "update_seconds": round(t1 - t0, 2), + "before": before_stats, "during": during, "integrity": integrity, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: all {replicas} replicas replaced one by one under load, " + f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_failover.py b/student-tests/run_failover.py new file mode 100644 index 00000000..0f28ec8c --- /dev/null +++ b/student-tests/run_failover.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys + +from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced + +HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + + +def ps(): + return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() + + +def aim(wanted): + """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" + probe = call("GET", "/health/live", 200).instance + if not any(container.startswith(probe) for container in wanted): + call("GET", "/health/live", 200) # теперь последней ответила другая реплика + + +def main(): + containers = api_containers() + check(len(containers) >= 2, "need at least 2 running api replicas") + print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") + stopped_once = set() + for attempt in range(1, 7): + wait_balanced(len(containers)) + if len(containers) == 2: + aim(set(containers) - stopped_once) + print(f"=== run {attempt}: api containers before") + print(ps()) + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public failover test failed") + print("api containers after") + print(ps()) + stopped = [container for container in containers if container not in api_containers()] + check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") + print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " + f"other api containers kept running, postgres/redis/lb were not touched") + stopped_once.add(stopped[0]) + docker("start", stopped[0]) + wait_balanced(len(containers)) + print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") + if stopped_once == set(containers): + break + check(stopped_once == set(containers), "not every replica was stopped by the public test") + print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_persistence.py b/student-tests/run_persistence.py new file mode 100644 index 00000000..66b3293b --- /dev/null +++ b/student-tests/run_persistence.py @@ -0,0 +1,47 @@ +import os +import subprocess +import sys +import time + +from common import BASE_URL, ROOT, check, compose_output, docker + +HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + + +def started(): + containers = compose_output("ps", "-q").split() + lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines() + return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)} + + +def main(): + before = started() + print("before: container, ID, started at") + for name, (cid, at) in sorted(before.items()): + print(f" {name:<28} {cid[:12]} {at}") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + begin = time.monotonic() + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + seconds = time.monotonic() - begin + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public persistence test failed") + after = started() + print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?") + restarted = [] + for name, (cid, at) in sorted(after.items()): + same_id = before.get(name, ("", ""))[0] == cid + new_start = before.get(name, ("", ""))[1] != at + print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}") + check(same_id, f"{name}: container was recreated, volumes/IDs must stay") + if new_start: + restarted.append(name.split("-")[-2]) + for service in ("postgres", "redis", "s3", "api", "janitor", "lb"): + check(service in restarted, f"{service} was not restarted by the hook") + print("PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, " + "new start time); data, the attachment bytes and the old token survived") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..7c770d9c --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,258 @@ +import base64 +import datetime as dt +import uuid + +from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send + + +def parse_time(value): + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + + +def test_duplicate_username(): + body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} + call("POST", "/api/v1/users", 201, body=body) + call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user")) + print("PASS: duplicate username returns 409") + + +def test_invalid_json(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", "/api/v1/users", 400, raw=b'{"username":') + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json", + headers={"Idempotency-Key": new_key()}) + print("PASS: invalid JSON returns 400") + + +def test_wrong_fields(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD}) + call("POST", "/api/v1/users", 400, + body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True}) + # Отправителя нельзя подставить: он берётся из сессии. + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()}) + print("PASS: wrong and extra fields return 400") + + +def test_foreign_chat(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + chat = new_chat(alice, bob) + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"]) + send(eve, chat["id"], "not my chat", expected=404) + # Несуществующий чат выглядит так же, как чужой. + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"]) + print("PASS: foreign chat returns 404") + + +def test_empty_message(): + alice = new_user("alice") + chat = new_chat(alice) + for text in ("", " ", "\n\t"): + send(alice, chat["id"], text, expected=400) + send(alice, chat["id"], "я" * 4001, expected=400) + send(alice, chat["id"], "я" * 4000) + print("PASS: empty and too long messages return 400, 4000 chars is ok") + + +def test_pagination(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)] + + history, pages = read_history(bob, chat["id"], limit=2) + check(pages == 4, f"expected 4 pages, got {pages}") + check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages") + + other = new_chat(alice) + for i in range(3): + send(alice, other["id"], f"Other {i}") + first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) + print("PASS: pagination works across multiple pages, foreign cursor returns 400") + + +def test_empty_cursor(): + alice = new_user("alice") + chat = new_chat(alice) + send(alice, chat["id"], "one message") + for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"): + for query in ("?cursor=", "?cursor", "?limit=10&cursor="): + call("GET", path + query, 400, token=alice["token"]) + call("GET", path, 200, token=alice["token"]) + print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page") + + +def test_chat_list_pagination(): + alice, bob = new_user("alice"), new_user("bob") + created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] + seen, cursor = [], None + while True: + query = "?limit=1" + (f"&cursor={cursor}" if cursor else "") + page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json + seen += [chat["id"] for chat in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + check(seen == created, "chat list pages are wrong") + print("PASS: chat list pagination") + + +def test_unicode(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob, title="Чат 👋") + text = "Привет 👋 你好" + check(send(alice, chat["id"], text).json["text"] == text, "text changed on write") + history, _ = read_history(bob, chat["id"]) + check(history[-1]["text"] == text, "text changed on read") + print("PASS: unicode text is stored as is") + + +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}, + headers={"Idempotency-Key": new_key()}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + + +def test_user_has_no_password(): + body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} + user = call("POST", "/api/v1/users", 201, body=body).json + check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}") + print("PASS: user response has no password or hash") + + +def test_password_rules(): + def register(password, expected): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password} + call("POST", "/api/v1/users", expected, body=body) + + register("x" * 11, 400) + register("x" * 129, 400) + register(" " * 12, 400) + register("x" * 12, 201) + register("я" * 128, 201) + print("PASS: password length 12..128, blank password is rejected") + + +def test_login_gives_token(): + username = "lg_" + uuid.uuid4().hex[:10] + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Login", "password": PASSWORD}).json + reply = login(username, PASSWORD) + session = reply.json + check(session["token_type"] == "Bearer", "token_type must be Bearer") + check(session["user"] == user, "session user differs from registered user") + check(len(session["token"]) >= 43, "token is too short") + check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque") + check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached") + left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s") + me = call("GET", "/api/v1/users/me", token=session["token"]).json + check(me == user, "/users/me returned another user") + print("PASS: Basic login returns opaque Bearer token with expiry") + + +def test_wrong_password(): + user = new_user("wp") + wrong = login(user["username"], "Wrong-password-000", expected=401) + unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) + check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") + check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") + print("PASS: wrong password and unknown user both return the same 401") + + +def test_malformed_basic(): + no_colon = base64.b64encode(b"just_username").decode() + for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"): + headers = {} if header is None else {"Authorization": header} + call("POST", "/api/v1/auth/sessions", 401, headers=headers) + print("PASS: malformed Basic header returns 401") + + +def test_password_with_colon_and_unicode(): + password = "пароль:с:двоеточием 👋" + user = new_user("colon", password=password) + call("GET", "/api/v1/users/me", token=user["token"]) + print("PASS: password with colon and unicode works") + + +def test_bearer_required(): + user = new_user("br") + for headers in ({}, {"Authorization": basic(user["username"], user["password"])}, + {"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}): + reply = call("GET", "/api/v1/users/me", 401, headers=headers) + check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer") + call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]}) + print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401") + + +def test_x_user_id_ignored(): + alice, bob = new_user("alice"), new_user("bob") + me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json + check(me["id"] == alice["id"], "X-User-Id changed the current user") + print("PASS: X-User-Id does not change the current user") + + +def test_auth_before_body(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) + print("PASS: 401 comes before 400 and 404") + + +def test_logout(): + user = new_user("lo") + second = login(user["username"], user["password"]).json["token"] + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + call("GET", "/api/v1/users/me", 401, token=user["token"]) + call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива + again = login(user["username"], user["password"]).json["token"] + check(again != user["token"], "new login must give a new token") + call("GET", "/api/v1/users/me", 200, token=again) + print("PASS: logout revokes only the current session") + + +if __name__ == "__main__": + run_tests([ + test_duplicate_username, + test_invalid_json, + test_wrong_fields, + test_foreign_chat, + test_empty_message, + test_pagination, + test_empty_cursor, + test_chat_list_pagination, + test_unicode, + test_nul_character_is_stored, + test_user_has_no_password, + test_password_rules, + test_login_gives_token, + test_wrong_password, + test_malformed_basic, + test_password_with_colon_and_unicode, + test_bearer_required, + test_x_user_id_ignored, + test_auth_before_body, + test_logout, + ]) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..e1178c12 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,268 @@ +"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ. + +Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа: +пользовательская ошибка не должна становиться 500. +""" +import hashlib +import http.client +import socket +import urllib.parse +import uuid +import zlib + +import images +from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload + +URL = urllib.parse.urlsplit(BASE_URL) + + +def setup(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + return alice, bob, eve, new_chat(alice, bob) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def check_error(reply, code): + check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}") + + +def test_png_and_jpeg_roundtrip(): + alice, bob, _, chat = setup() + for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"), + (images.sample_jpeg(), "image/jpeg", "sample.jpg"), + (images.progressive_jpeg(), "image/jpeg", "progressive.jpg"), + (images.png(33, 17, interlace=1), "image/png", "interlaced.png"), + (images.png(1, 1, gray=True), "image/png", "1x1.png")): + att = upload(alice, chat["id"], data, ctype, name).json + check(att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None, "not ready") + check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ") + check(att["content_type"] == ctype and att["filename"] == name, "metadata differs") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta == att, "GET metadata differs from POST response") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "downloaded bytes differ") + check(got.headers["content-type"] == ctype, "wrong Content-Type on download") + check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length") + check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing") + print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes") + print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte") + + +def test_size_limit_exact_boundary(): + alice, _, _, chat = setup() + exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES) + check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB") + att = upload(alice, chat["id"], exact).json + check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed") + print(f"exactly {len(exact)} bytes (10 MiB) -> 201") + over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1) + reply = upload(alice, chat["id"], over, expected=413) + check_error(reply, "payload_too_large") + print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body") + print("PASS: the 10 MiB limit is exact") + + +def chunked_upload(user, chat_id, total, piece=64 * 1024): + """Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении.""" + conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60) + + def body(): + sent = 0 + head = images.sample_png()[:64] + yield head + sent += len(head) + while sent < total: + part = bytes(min(piece, total - sent)) + sent += len(part) + yield part + try: + conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(), + headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]}, + encode_chunked=True) + reply = conn.getresponse() + return reply.status, reply.read() + finally: + conn.close() + + +def test_size_limit_streaming(): + alice, _, _, chat = setup() + status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024) + check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}") + print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed") + call("GET", "/health/ready", 200) + print("PASS: oversized streamed body is cut off, the API keeps working") + + +def test_pixel_limit(): + alice, _, _, chat = setup() + at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей + att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json + print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 201 {att['id'][:8]}") + over = images.png(8001, 5000, gray=True) + check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large") + print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413") + bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01") + check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large") + print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded") + print("PASS: pixel limit is checked before decompression") + + +def test_damaged_images(): + alice, _, _, chat = setup() + png, jpeg = images.sample_png(), images.sample_jpeg() + cases = [ + ("PNG cut in half", png[:len(png) // 2], "image/png"), + ("PNG without IEND", png[:-12], "image/png"), + ("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"), + ("PNG with garbage after IEND", png + b"garbage", "image/png"), + ("PNG signature only", png[:8], "image/png"), + ("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"), + ("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"), + ("JPEG header only", jpeg[:20], "image/jpeg"), + ("random bytes", uuid.uuid4().bytes * 100, "image/png"), + ("plain text", "это не картинка".encode(), "image/jpeg"), + ] + for name, data, ctype in cases: + reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415) + check_error(reply, "unsupported_media_type") + print(f"{name} ({len(data)} bytes, {ctype}) -> 415") + check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request") + print("empty body -> 400") + print("PASS: damaged and foreign files are rejected with 415, never 500") + + +def test_mime_substitution(): + alice, _, _, chat = setup() + cases = [ + ("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"), + ("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"), + ("GIF bytes sent as image/png", images.GIF, "image/png"), + ("PNG sent as image/gif", images.sample_png(), "image/gif"), + ("PNG sent as text/html", images.sample_png(), "text/html"), + ("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"), + ("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"), + ] + for name, data, ctype in cases: + check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type") + print(f"{name} -> 415") + att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json + check(att["content_type"] == "image/png", "content type must be normalised") + print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png") + print("PASS: the type is taken from the content; a declared type that does not match is refused") + + +def test_filenames(): + alice, _, _, chat = setup() + bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png", + "bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256] + for name in bad: + check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request") + print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400") + check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request") + print("no filename -> 400") + for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"): + att = upload(alice, chat["id"], images.sample_png(), filename=name).json + check(att["filename"] == name, "display name changed") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded") + print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only") + print("PASS: filename never becomes a path; dangerous names are refused") + + +def test_access_control(): + alice, bob, eve, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + for suffix in ("", "/content"): + reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + check_error(reply, "not_found") + call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400") + check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found") + check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found") + upload(None, chat["id"], images.sample_png(), expected=401) + print("upload into someone else's / unknown chat -> 404, without token -> 401") + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", 400, token=bob["token"]) + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=original", 200, token=bob["token"]) + print("variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200") + print("PASS: only chat members can see an attachment") + + +def test_message_attachments(): + alice, bob, eve, chat = setup() + first = upload(alice, chat["id"], images.sample_png()).json + second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json + msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json + check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed") + print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member") + plain = post_message(bob, chat["id"], "без вложений").json + check(plain["attachment_ids"] == [], "default attachment_ids must be []") + reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json + history, _ = read_history(bob, chat["id"]) + check(history == [msg, plain, reused], "history differs from the confirmed messages") + print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed") + + other_chat = new_chat(alice, eve) + foreign = upload(alice, other_chat["id"], images.sample_png()).json + for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"), + ([first["id"], foreign["id"]], "own + foreign attachment"), + ([str(uuid.uuid4())], "unknown attachment_id")): + check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found") + print(f"{what} -> 404") + check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found") + print("outsider referencing an attachment of the chat -> 404") + for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"), + ([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"), + (["not-a-uuid"], "x", "bad UUID"), + ([], "", "empty text without attachments"), + ([], " ", "whitespace text without attachments")): + check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request") + print(f"{what} -> 400") + history_after, _ = read_history(bob, chat["id"]) + check(history_after == history, "a rejected message was stored") + print("PASS: only attachments of the same chat can be linked; rejected requests change nothing") + + +def test_idempotency_with_attachments(): + alice, bob, _, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json + key = new_key() + first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + check(again == first, "replay returned a different message") + check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict") + check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict") + key2 = new_key() + plain = post_message(alice, chat["id"], "текст", None, key=key2).json + check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field") + history, _ = read_history(bob, chat["id"]) + check(len(history) == 2, "replays created extra messages") + print("same key + same attachments -> same 201; other attachments -> 409; missing field == []") + print("PASS: Idempotency-Key covers attachment_ids") + + +def test_aborted_upload(): + alice, _, _, chat = setup() + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000)) + sock.close() # клиент пропал на середине тела + print("sent 1.1 MB of a declared 5 MB body and closed the connection") + att = upload(alice, chat["id"], images.sample_png()).json + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + print("the API keeps serving uploads; nothing of the aborted request is visible " + "(DB/S3 side is checked by test_consistency.py)") + print("PASS: an interrupted upload is dropped") + + +if __name__ == "__main__": + run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming, + test_pixel_limit, test_damaged_images, test_mime_substitution, test_filenames, test_access_control, + test_message_attachments, test_idempotency_with_attachments, test_aborted_upload]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 00000000..01f2251c --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,103 @@ +from collections import Counter + +import images +from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload + + +def test_requests_spread_over_replicas(): + alice = new_user("alice") + replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)] + served = Counter(reply.instance for reply in replies) + print(f"20 x GET /users/me with one token -> replicas: {dict(served)}") + check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned") + check(len(served) >= 2, "all requests went to one replica: balancing does not work") + print("PASS: one session works on every replica, no second login (sessions are shared in Redis)") + + +def test_write_on_one_read_on_other(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = send(alice, chat["id"], "written on one replica") + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]) + check(sent.json in reply.json["items"], "confirmed message is not visible on another replica") + readers.add(reply.instance) + print(f"message written by {sent.instance}; read back by {sorted(readers)}") + check(readers - {sent.instance}, "no read went to another replica") + print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)") + + +def test_logout_seen_by_all_replicas(): + user = new_user("logout") + out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)] + print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}") + check(len(set(after)) >= 2, "only one replica answered") + print("PASS: logout on one replica revokes the session on all of them") + + +def test_idempotent_retry_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "retry me", key=key) + for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику + retry = send(alice, chat["id"], "retry me", key=key) + if retry.instance != first.instance: + break + print(f"first POST on {first.instance} -> 201 {first.json['id']}; " + f"same key on {retry.instance} -> {retry.status} {retry.json['id']}") + check(retry.instance != first.instance, "retry never reached another replica") + check(retry.json == first.json, "retry on another replica returned a different message") + items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"] + check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate") + print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message") + + +def test_attachment_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + sent = upload(alice, chat["id"], data) + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"]) + check(reply.data == data, "another replica returned different bytes") + readers.add(reply.instance) + print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}") + check(readers - {sent.instance}, "no download went to another replica") + print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)") + + +def test_instance_header_everywhere(): + containers = api_containers() + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + replies = [ + call("GET", "/health/live", 200), + call("GET", "/health/ready", 200), + send(alice, chat["id"], "hello"), + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}), + call("GET", "/api/v1/users/me", 401), + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]), + call("POST", "/api/v1/users", 409, + body={"username": alice["username"], "display_name": "x", "password": alice["password"]}), + ] + for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер + check(any(container.startswith(reply.instance) for container in containers), + f"X-Instance-Id {reply.instance} is not a running api container") + print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container " + f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}") + print("PASS: X-Instance-Id is in every response and points to a real container") + + +if __name__ == "__main__": + run_tests([ + test_requests_spread_over_replicas, + test_write_on_one_read_on_other, + test_logout_seen_by_all_replicas, + test_idempotent_retry_on_other_replica, + test_attachment_on_other_replica, + test_instance_header_everywhere, + ]) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 00000000..af10ad17 --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,275 @@ +"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка). + +Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`), +выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает +janitor вручную. Данные синтетические, volumes не удаляются. + + python student-tests/test_consistency.py [s3|crash|orphan|aborted|all] +""" +import json +import socket +import sys +import time +import urllib.parse + +import images +from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat, + new_user, post_message, run_tests, sql_value, upload, wait_ready) + +URL = urllib.parse.urlsplit(BASE_URL) + +# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но finish_upload +# (UPDATE ... state='ready' + COMMIT) падает — как если бы реплика умерла сразу после PUT в S3. +CRASH_AFTER_PUT = r''' +import json, struct, sys, tempfile, zlib +from uuid import UUID +import attachments, db, storage, uploads +chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2]) +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b"")) +f = tempfile.SpooledTemporaryFile(); f.write(data) +import hashlib +upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest()) +image = uploads.inspect(upload, "image/png") +created = {} +real_intent = db.create_upload_intent +def remember(attachment_id, *args): + created["id"], created["key"] = str(attachment_id), args[-1] + return real_intent(attachment_id, *args) +def crash(attachment_id): + raise RuntimeError("simulated crash between S3 PUT and DB COMMIT") +db.create_upload_intent = remember +db.finish_upload = crash +db.pool.open() +try: + attachments.store(chat_id, owner_id, "crash.png", upload, image) + outcome = "stored" +except RuntimeError as exc: + outcome = str(exc) +print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"], + "object_exists": storage.exists(created["key"])})) +''' + +OBJECT_EXISTS = r''' +import json, sys, storage +print(json.dumps({"exists": storage.exists(sys.argv[1])})) +''' + +PUT_ORPHAN = r''' +import hashlib, io, json, uuid, storage +data = b"orphan object without a row in PostgreSQL" +key = storage.new_key(uuid.uuid4().hex) +storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest()) +print(json.dumps({"key": key})) +''' + +DELETE_OBJECT = r''' +import json, sys, storage +storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]})) +''' + +COUNT_OBJECTS = r''' +import json, storage +print(json.dumps({"objects": sum(1 for _ in storage.list_objects())})) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def object_exists(key): + return last_json(app_python(OBJECT_EXISTS, key))["exists"] + + +def backdate(where): + sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}") + + +def uploading_rows(chat_id): + return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'")) + + +def wait_download(user, att, seconds=60): + deadline = time.monotonic() + seconds + while True: + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"]) + if reply.status == 200: + return reply + check(time.monotonic() < deadline, "download did not recover after S3 start") + time.sleep(1) + + +def s3_outage(stop_action, start_action, label): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + att = upload(alice, chat["id"], data).json + before = uploading_rows(chat["id"]) + print(f"--- S3 {label}: docker compose {stop_action} s3") + compose(stop_action, "s3") + try: + started = time.monotonic() + reply = upload(alice, chat["id"], data, expected=503) + ms = (time.monotonic() - started) * 1000 + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + body = reply.data.decode().lower() + check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")), + "error leaks storage details") + print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body") + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + print("download of an existing file -> 503") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta["sha256"] == att["sha256"], "metadata changed") + post_message(alice, chat["id"], "текст без файла") + post_message(alice, chat["id"], "", [att["id"]]) + call("GET", "/health/ready", 200) + print("metadata, messages (also with an already stored attachment) and /health/ready keep working: " + "S3 is needed only for file bytes") + after = uploading_rows(chat["id"]) + print(f"uploading rows of this chat: before {before}, after the failed upload {after} " + "(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API") + finally: + print(f"docker compose {start_action} s3") + compose(start_action, "s3") + got = wait_download(bob, att) + check(got.data == data, "file changed after S3 recovery") + print("after S3 recovery the old file downloads byte for byte") + upload(alice, chat["id"], data) + print("new upload -> 201") + leftover = uploading_rows(chat["id"]) + if leftover: + backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'") + report = janitor() + print(f"janitor after grace period: {json.dumps(report)}") + check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain") + print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up") + + +def test_s3_stopped(): + s3_outage("stop", "start", "stopped (connection refused / name not resolved)") + + +def test_s3_frozen(): + s3_outage("pause", "unpause", "frozen (no answer, timeouts)") + + +def test_crash_between_s3_and_db(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"])) + check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}") + att_id, key = result["attachment_id"], result["object_key"] + state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'") + check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3") + print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3") + call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"]) + post_message(alice, chat["id"], "x", [att_id], expected=404) + print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404") + + report = janitor() + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key), + "fresh upload was deleted inside the grace period") + print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> " + "fresh row and object are kept (the upload might still be running)") + backdate(f"id = '{att_id}'") + report = janitor("--dry-run") + check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row") + print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted") + report = janitor() + check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists") + check(not object_exists(key), "object still exists") + print(f"janitor: {json.dumps(report)} -> row and object deleted") + report = janitor() + print(f"janitor again: {json.dumps(report)} -> idempotent") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment damaged by cleanup") + print("the ready attachment of the same chat is untouched") + print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period") + + +def test_orphan_objects(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + key = last_json(app_python(PUT_ORPHAN))["key"] + print(f"object {key} put into the bucket without a row in PostgreSQL") + # Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с. + short = {"UPLOAD_TIMEOUT_SECONDS": "1"} + report = janitor("--grace-seconds", "3", env=short) + check(object_exists(key), "fresh orphan deleted inside grace period") + print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept") + time.sleep(4) + report = janitor("--grace-seconds", "3", "--dry-run", env=short) + check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key), + "dry run must report and keep the orphan") + print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept") + report = janitor("--grace-seconds", "3", env=short) + check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted") + print(f"janitor: {json.dumps(report)} -> orphan deleted") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment was deleted with orphans") + print("ready attachments older than the grace period are untouched (they have rows)") + + victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json + victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'") + app_python(DELETE_OBJECT, victim_key) + report = janitor("--grace-seconds", "3", env=short) + check(report["missing_objects"] >= 1, "missing object not reported") + check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted") + reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"]) + print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects=" + f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)") + print("PASS: orphans are found and deleted only after the grace period; missing objects are reported") + + +def api_tmp_files(): + counts = {} + for container in compose_output("ps", "-q", "api").split(): + counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0) + return counts + + +def test_aborted_upload_leaves_nothing(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + objects = last_json(app_python(COUNT_OBJECTS))["objects"] + rows = int(sql_value("SELECT count(*) FROM attachments")) + for sent in (100_000, 3_000_000): + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n" + f"Content-Length: 9000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(sent)) + sock.close() + print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection") + time.sleep(2) + check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row") + check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object") + print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}") + tmp = api_tmp_files() + check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}") + print(f"temporary upload files in api containers: {tmp} (removed when the request ends)") + for container in compose_output("ps", "-q", "api").split(): + mounts = docker("inspect", "-f", "{{json .Mounts}}", container) + # файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных + storage = [m for m in json.loads(mounts) + if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)] + check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}") + print("api containers have no volumes: the original is never stored locally") + print("PASS: an interrupted upload leaves no row, no object and no local file") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db], + "orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]} + groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"] + check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]") + wait_ready() + run_tests(groups[target]) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 00000000..3d71fcc1 --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,101 @@ +import time +import uuid + +from common import call, check, compose_output, new_user, psql, run_tests + + +def sql_value(sql): + code, output, error = psql(sql) + check(code == 0, f"psql failed: {error}") + return output + + +def test_create_chat_rolls_back(): + alice = new_user("rollback") + title = "rollback-" + uuid.uuid4().hex[:8] + inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + call("POST", "/api/v1/chats", 400, token=alice["token"], + body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) + + left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") + orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") + inserted_after = inserted_before + deadline = time.monotonic() + 15 + while inserted_after == inserted_before and time.monotonic() < deadline: + time.sleep(0.5) + inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; " + f"chats insert counter: {inserted_before} -> {inserted_after}") + check(left == "0" and orphans == "0", "chat row survived the failed transaction") + check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown") + print("PASS: chat row was inserted and then rolled back together with members") + + +def test_message_needs_membership_in_db(): + code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") + print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") + check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") + print("PASS: the database itself rejects a message from a non-member") + + +def test_password_is_argon2id(): + user = new_user("hash") + stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'") + print(f"stored password: {stored[:31]}... (length {len(stored)})") + check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash") + print("PASS: password is stored as Argon2id hash") + + +def test_unique_username_constraint(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'users'::regclass AND contype = 'u'") + print(f"unique constraint on users: {constraint}") + check(constraint == "UNIQUE (username)", "no unique constraint on username") + print("PASS: username is unique in the database") + + +def test_migrations_are_not_repeated(): + versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + print(f"applied migrations: {versions}; running the migrate service once more") + output = compose_output("run", "--rm", "--no-deps", "-T", "migrate") + again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1] + print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}") + check(again == versions and "none" in last_log, "migrations were applied again") + print("PASS: migrations are applied once, a repeated run does nothing") + + +def test_replicas_do_not_migrate(): + """lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему.""" + logs = compose_output("logs", "api").splitlines() + checked = [line for line in logs if "Schema is up to date" in line] + migrated = [line for line in logs if "Migrations applied" in line] + migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': " + f"{len(migrated)}; migrate service: {migrate_exit}") + check(checked and not migrated, "api replicas must not apply migrations") + check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0") + print("PASS: only the migrate service applies migrations, replicas wait for the schema") + + +def test_idempotency_key_is_unique_in_db(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conname = 'messages_idempotency_key'") + print(f"unique constraint for Idempotency-Key: {constraint}") + check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys") + print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database") + + +if __name__ == "__main__": + run_tests([ + test_create_chat_rolls_back, + test_message_needs_membership_in_db, + test_password_is_argon2id, + test_unique_username_constraint, + test_migrations_are_not_repeated, + test_replicas_do_not_migrate, + test_idempotency_key_is_unique_in_db, + ]) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 00000000..e6017ce1 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,57 @@ +import datetime as dt +import os +import time + +from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready + +SHORT_TTL = 2 + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env + else: + env["SESSION_TTL_SECONDS"] = str(ttl) + compose("up", "-d", "--no-deps", "--wait", "api", env=env) + wait_ready() + + +def seconds_left(expires_at): + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def test_session_expires(): + print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}") + restart_api(SHORT_TTL) + try: + user = new_user("ttl") + session = login(user["username"], user["password"]).json + token = session["token"] + left = seconds_left(session["expires_at"]) + print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: " + f"{redis_cli('pttl', session_key(token))} ms") + check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions") + + call("GET", "/api/v1/users/me", 200, token=token) + print("before expiry: GET /users/me -> 200") + + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0) + call("GET", "/api/v1/users/me", 401, token=token) + exists = redis_cli("exists", session_key(token)) + print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}") + check(exists == "0", "Redis must delete the expired session key") + finally: + print("recreating api with SESSION_TTL_SECONDS from .env") + restart_api() + + user = new_user("ttl_back") + left = seconds_left(login(user["username"], user["password"]).json["expires_at"]) + print(f"normal API again: new session expires in {left:.0f} s") + check(left > 300, "normal TTL was not restored") + print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401") + + +if __name__ == "__main__": + run_tests([test_session_expires]) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 00000000..8eaef386 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,110 @@ +import sys +import time +import uuid + +from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready + +FAKE_TOKEN = "A" * 43 + + +def timed(method, path, expected, **kwargs): + started = time.monotonic() + reply = call(method, path, expected, **kwargs) + return reply, (time.monotonic() - started) * 1000 + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + before = send(alice, chat["id"], "confirmed before outage").json + return alice, bob, chat, before + + +def recovered(alice, chat, before): + wait_ready() + history, _ = read_history(alice, chat["id"]) + check(before in history, "message confirmed before the outage is lost or changed") + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the old token works, confirmed messages are in place") + + +def test_redis_down(): + alice, bob, chat, before = prepare() + print("stopping redis") + compose("stop", "redis") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") + call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) + print("unknown token -> 503, not 200: nobody is trusted without a check") + call("GET", "/api/v1/chats", 401) + print("no token -> 401") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + print("login -> 503: there is nowhere to store the session") + call("POST", "/api/v1/users", 201, + body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("registration -> 201: it needs only PostgreSQL") + finally: + print("starting redis") + compose("start", "redis") + recovered(alice, chat, before) + print("PASS: Redis outage gives 503 and does not let anyone in without a session check") + + +def test_redis_crash(): + alice = new_user("crash") + print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") + compose("kill", "redis") + try: + # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. + status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status + print(f"right after the crash: GET /users/me -> {status}") + finally: + print("starting redis") + compose("start", "redis") + wait_ready() + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the session issued before the crash still works (AOF, appendfsync always)") + print("PASS: sessions survive a Redis crash, not only a clean restart") + + +def test_postgres_down(): + alice, bob, chat, before = prepare() + print("stopping postgres") + compose("stop", "postgres") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + for method, path, body in (("GET", "/api/v1/users/me", None), + ("GET", "/api/v1/chats", None), + ("GET", f"/api/v1/chats/{chat['id']}/messages", None), + ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): + reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + check("postgres" not in reply.data.decode().lower(), "error must not show database details") + print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") + call("GET", "/api/v1/chats", 401) + call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) + print("no token -> 401, unknown token -> 401: anonymous users are not let in") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + call("POST", "/api/v1/users", 503, + body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("login -> 503, registration -> 503") + finally: + print("starting postgres") + compose("start", "postgres") + recovered(alice, chat, before) + print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], + "all": [test_redis_down, test_redis_crash, test_postgres_down]} + check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") + run_tests(tests[target]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..407dea3d --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,174 @@ +import json +import socket +import threading +import time +import urllib.parse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send + +PARALLEL = 20 + + +def post_raw(user, chat_id, raw, key): + """POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"], + raw=raw, headers={"Idempotency-Key": key}) + + +def texts_in(user, chat_id): + history, _ = read_history(user, chat_id) + return [message["text"] for message in history] + + +def test_replay_returns_same_message(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "Привет 👋", key=key) + # То же JSON-значение, записанное иначе: пробелы и \u-экранирование. + same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key) + again = send(alice, chat["id"], "Привет 👋", key=key) + print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}") + check(same.status == 201 and same.json == first.json and again.json == first.json, + "retry must return the original 201 and the same message") + check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate") + print("PASS: a retry with the same key returns the original 201 and message, no duplicate") + + +def test_same_key_parallel_through_replicas(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(_): + barrier.wait() # все запросы стартуют одновременно + return send(alice, chat["id"], "one message for 20 requests", key=key) + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + replies = list(pool.map(post, range(PARALLEL))) + bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies} + served = Counter(reply.instance for reply in replies) + count = texts_in(bob, chat["id"]).count("one message for 20 requests") + print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, " + f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}") + check(len(served) >= 2, "requests did not go through different replicas") + check(len(bodies) == 1 and count == 1, "parallel retries created more than one message") + print("PASS: parallel requests with one key through different replicas create exactly one message") + + +def test_same_key_different_bodies_parallel(): + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(index): + text = "body A" if index % 2 == 0 else "body B" + barrier.wait() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"], + body={"text": text}, headers={"Idempotency-Key": key}) + return text, reply + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(post, range(PARALLEL))) + texts = texts_in(alice, chat["id"]) + winner = [t for t in texts if t in ("body A", "body B")] + print(f"{PARALLEL} parallel POSTs, one key, two different bodies: " + f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}") + check(len(winner) == 1, "exactly one message must be created") + for text, reply in results: + expected = 201 if text == winner[0] else 409 + check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}") + print("PASS: the first body wins, the same body gets 201, the other body gets 409") + + +def test_key_scope(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + other = new_chat(alice, bob) + key = new_key() + a1 = send(alice, chat["id"], "scope", key=key).json + b1 = send(bob, chat["id"], "scope", key=key).json + a2 = send(alice, other["id"], "scope", key=key).json + print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages") + check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat") + call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"], + body={"text": "other text"}, headers={"Idempotency-Key": key}) + print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409") + + +def test_key_validation(): + alice = new_user("alice") + chat = new_chat(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": "no key"}) + for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"): + call("POST", path, 400, token=alice["token"], body={"text": "bad key"}, + headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8 + longest = ("Az09._:-" * 16)[:128] + call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest}) + print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201") + print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}") + + +def test_failed_request_does_not_burn_key(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key}) + call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key}) + created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key}) + print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}") + print("PASS: rejected requests (400, 404) do not store a result for the key") + + +def test_replay_checks_session_and_access(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + send(alice, chat["id"], "private", key=key) + stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}") + print("PASS: a retry checks the session and chat access again, the saved answer is not leaked") + + +def test_lost_response_then_retry(): + """Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен.""" + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + body = json.dumps({"text": "answer was lost"}).encode("utf-8") + url = urllib.parse.urlsplit(BASE_URL) + request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection: + connection.sendall(request + body) # и сразу закрываем, ответ не читаем + time.sleep(1) + retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"], + body={"text": "answer was lost"}, headers={"Idempotency-Key": key}) + count = texts_in(alice, chat["id"]).count("answer was lost") + print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}") + check(count == 1, "the retry after a lost answer created a duplicate") + print("PASS: an unconfirmed POST can be safely retried with the same key") + + +if __name__ == "__main__": + run_tests([ + test_replay_returns_same_message, + test_same_key_parallel_through_replicas, + test_same_key_different_bodies_parallel, + test_key_scope, + test_key_validation, + test_failed_request_does_not_burn_key, + test_replay_checks_session_and_access, + test_lost_response_then_retry, + ]) diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 00000000..4af44fad --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,57 @@ +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send + +CLIENTS = 10 +MESSAGES_PER_CLIENT = 10 +REGISTRATIONS = 20 + + +def test_concurrent_messages(): + users = [new_user(f"race{i}") for i in range(CLIENTS)] + chat = new_chat(*users, title="race") + barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно + + def client(index): + barrier.wait() + for n in range(MESSAGES_PER_CLIENT): + send(users[index], chat["id"], f"client-{index}-message-{n}") + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков + elapsed = time.monotonic() - started + + history, pages = read_history(users[0], chat["id"], limit=30) + ids = {message["id"] for message in history} + expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)} + print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; " + f"read {len(history)} in {pages} pages, unique ids: {len(ids)}") + check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages") + check(len(ids) == len(history), "duplicate message ids") + check({message["text"] for message in history} == expected, "lost or changed message texts") + print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates") + + +def test_concurrent_registration(): + username = "race_" + uuid.uuid4().hex[:10] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + barrier.wait() + body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD} + return call("POST", "/api/v1/users", (201, 409), body=body).status + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + statuses = list(pool.map(register, range(REGISTRATIONS))) + created, conflicts = statuses.count(201), statuses.count(409) + print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}") + check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win") + print("PASS: concurrent registration created exactly one user") + + +if __name__ == "__main__": + run_tests([test_concurrent_messages, test_concurrent_registration]) diff --git a/student-tests/test_s3_security.py b/student-tests/test_s3_security.py new file mode 100644 index 00000000..ece71a30 --- /dev/null +++ b/student-tests/test_s3_security.py @@ -0,0 +1,146 @@ +"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен. + +Нужен запущенный стенд из compose.yaml. +""" +import json +import socket + +import images +from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload + +# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API. +ANONYMOUS = r''' +import json, sys, urllib.request, urllib.error +import settings +base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET +def status(method, url, data=None): + try: + with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r: + return r.status, r.read(300).decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read(300).decode("utf-8", "replace") +out = {} +for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None), + ("head_object", "HEAD", base + "/" + sys.argv[1], None), + ("list_bucket", "GET", base + "?list-type=2", None), + ("put_object", "PUT", base + "/originals/anonymous-test", b"x"), + ("get_bucket_policy", "GET", base + "?policy", None)): + code, body = status(method, url, data) + out[name] = {"status": code, "s3_code": body.split("")[1].split("")[0] if "" in body else ""} +print(json.dumps(out)) +''' + +# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json). +API_KEY_RIGHTS = r''' +import io, json, sys, uuid +from botocore.exceptions import ClientError +import settings, storage +c, bucket = storage._client, settings.S3_BUCKET +def attempt(name, fn): + try: + fn(); return "allowed" + except ClientError as e: + return "denied: " + e.response.get("Error", {}).get("Code", "?") +public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) +key = "originals/" + uuid.uuid4().hex +result = { + "get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()), + "put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")), + "delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)), + "list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)), + "create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])), + "make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)), + "read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)), + "delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)), + "change versioning": attempt("versioning", lambda: c.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})), +} +try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket + result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", [])) +except ClientError as e: + result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?") +print(json.dumps(result)) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + att = upload(alice, chat["id"], images.sample_png()).json + key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + return alice, bob, att, key + + +def test_bucket_is_private(): + alice, bob, att, key = prepare() + result = last_json(app_python(ANONYMOUS, key)) + for name, reply in result.items(): + print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}") + check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}") + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + print("the same file through the API for a chat member -> 200") + print("PASS: the bucket is private: no anonymous read, list, write or policy read") + + +def test_api_key_is_limited(): + alice, bob, att, key = prepare() + result = last_json(app_python(API_KEY_RIGHTS, key)) + allowed = {"get own object", "put object", "delete object", "list bucket"} + visible = result.pop("visible buckets") + print(f"API key: {'list buckets':<32} -> {visible}") + check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets") + for name, outcome in result.items(): + print(f"API key: {name:<32} -> {outcome}") + if name in allowed: + check(outcome == "allowed", f"API key must be allowed to {name}") + else: + check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}") + print("PASS: the API key can only get/put/delete/list objects of its own bucket; " + "bucket admin, policy and other buckets are denied") + + +def test_no_open_ports_or_console(): + config = json.loads(compose_output("config", "--format", "json")) + s3 = config["services"]["s3"] + check(not s3.get("ports"), "s3 must not publish ports") + check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, " + "filer UI 8888 are reachable only inside the stand)") + for port in (8333, 9333, 8888, 19333): + sock = socket.socket() + sock.settimeout(1) + refused = sock.connect_ex(("127.0.0.1", port)) != 0 + sock.close() + check(refused, f"127.0.0.1:{port} is reachable from the host") + print(f"127.0.0.1:{port} from the host -> connection refused") + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()} + for name, items in sorted(secrets.items()): + print(f"secrets of {name}: {items or '-'}") + for name in ("api", "janitor"): + check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"}, + f"{name} must not get S3 admin keys or the key config") + check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key") + print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init") + + +def test_bootstrap_is_idempotent(): + alice, bob, att, key = prepare() + output = compose_output("run", "--rm", "--no-deps", "s3-init") + print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip())) + check("bucket is private" in output, "bootstrap did not confirm a private bucket") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "file changed after repeated bootstrap") + upload(alice, att["chat_id"], images.sample_png()) + print("after the repeated bootstrap: old file downloads, new upload works") + print("PASS: bucket/user/policy bootstrap can be repeated safely") + + +if __name__ == "__main__": + run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console, + test_bootstrap_is_idempotent]) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..42dcb5e9 --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,218 @@ +"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик. + +Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что +тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково +(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше +DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование) +при тех же 10 загрузках не помещается в лимит 512 MiB. + +Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков +одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status +(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера +(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с. + + python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json] +""" +import argparse +import hashlib +import json +import platform +import statistics +import threading +import time + +import images +from common import ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_balanced, wait_ready + + +def proc_status(container): + """VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты).""" + script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; " + "for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current " + "/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; " + "do [ -r $f ] && echo \"$f $(cat $f)\"; done; true") + out = docker("exec", container, "sh", "-c", script) + values = {} + for line in out.splitlines(): + parts = line.replace(":", " ").split() + if parts[0] in ("VmRSS", "VmHWM"): + values[parts[0]] = int(parts[1]) + elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")): + values["cgroup_peak"] = int(parts[1]) + elif parts[0].endswith(("memory.current", "usage_in_bytes")): + values["cgroup_current"] = int(parts[1]) + return values + + +class Sampler(threading.Thread): + def __init__(self, containers): + super().__init__(daemon=True) + self.containers = containers + self.stop = threading.Event() + self.rss = {c: 0 for c in containers} + self.cgroup = {c: 0 for c in containers} + self.samples = 0 + + def run(self): + while not self.stop.is_set(): + for c in self.containers: + try: + v = proc_status(c) + except Exception: # noqa: BLE001 — один пропущенный замер не важен + continue + self.rss[c] = max(self.rss[c], v.get("VmRSS", 0)) + self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0)) + self.samples += 1 + + +def api_containers(): + from common import api_containers as running + return running() + + +def fresh_replicas(): + compose("restart", "api") + containers = wait_balanced(2, seconds=90) + wait_ready() + return containers + + +def warm_up(user, chat): + """По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п.""" + for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4: + upload(user, chat["id"], data, ctype) + + +def scenario(name, payloads, concurrency): + user = new_user("mem") + chat = new_chat(user) + containers = fresh_replicas() + warm_up(user, chat) + before = {c: proc_status(c) for c in containers} + sampler = Sampler(containers) + sampler.start() + barrier = threading.Barrier(concurrency) + results = [None] * concurrency + + def worker(i): + data, ctype = payloads[i % len(payloads)] + barrier.wait() + started = time.monotonic() + try: + reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180) + results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data) + except AssertionError as exc: + results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)] + wall = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - wall + time.sleep(0.5) + sampler.stop.set() + sampler.join() + after = {c: proc_status(c) for c in containers} + + ok = [r for r in results if r[0] == 201] + check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 201]}") + for status, _, _, att, data in ok[:3]: + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"]) + check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs") + latencies = sorted(r[1] for r in results) + per_replica = [] + for c in containers: + b, a = before[c], after[c] + per_replica.append({ + "replica": c[:12], + "uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])), + "rss_before_mib": round(b["VmRSS"] / 1024, 1), + "rss_peak_mib": round(a["VmHWM"] / 1024, 1), + "rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1), + "rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1), + "cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None, + "cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None, + }) + sizes = sorted({len(d) for d, _ in payloads}) + summary = { + "scenario": name, + "concurrency": concurrency, + "file_bytes": sizes, + "total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1), + "all_201": True, + "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)}, + "sampler_rounds": sampler.samples, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False)) + return summary + + +# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование. +NAIVE_VS_STREAMING = r''' +import json, os, subprocess, sys +code = r""" +import io, os, resource, struct, sys, zlib +sys.path.insert(0, os.getcwd()) +os.environ.setdefault("S3_ACCESS_KEY_ID", "x") +from PIL import Image +import uploads # импорт до замера: сравниваем только саму проверку файла +Image.MAX_IMAGE_PIXELS = None +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +w, h = 8000, 5000 +comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3) +idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush() +data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") +del idat, comp, row +base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +if sys.argv[1] == "naive": + body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB) + im = Image.open(io.BytesIO(data)); im.load() +else: + f = io.BytesIO(data); uploads.check_png(f) +print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024) +""" +out = {} +for mode in ("naive", "streaming"): + r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd()) + out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:] +print(json.dumps(out)) +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default="evidence/upload-memory.json") + args = parser.parse_args() + check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads") + + print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}") + print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...") + large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)] + pixels = [(images.png(8000, 5000), "image/png")] + jpeg = [(images.sample_jpeg(), "image/jpeg")] + print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes") + + runs = [ + scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency), + scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency), + scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency), + ] + naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1]) + print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) " + f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB") + limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split() + result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20, + "api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive} + (ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"]) + print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB " + f"(limit {result['api_limit_mib']} MiB); results in {args.output}") + + +if __name__ == "__main__": + main() diff --git a/uploads.py b/uploads.py new file mode 100644 index 00000000..24522125 --- /dev/null +++ b/uploads.py @@ -0,0 +1,351 @@ +"""Приём и проверка загружаемого изображения с ограниченной памятью. + +1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше + MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном + файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса. +2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях + читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково + (CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма), + JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение. +""" +import asyncio +import hashlib +import logging +import struct +import tempfile +import threading +import time +import unicodedata +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image +from starlette.requests import ClientDisconnect, Request + +import settings +from errors import bad_request, payload_too_large, unsupported_media_type + +log = logging.getLogger("messenger.uploads") + +PNG = "image/png" +JPEG = "image/jpeg" +ALLOWED_TYPES = (PNG, JPEG) +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" + +SPOOL_IN_MEMORY = 256 * 1024 +INFLATE_CHUNK = 64 * 1024 +CRC_CHUNK = 64 * 1024 + +# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке). +Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS + +# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY. +_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY) + + +@dataclass +class Upload: + file: BinaryIO + size: int + sha256: str + + +@dataclass +class ImageInfo: + content_type: str + width: int + height: int + + +class Damaged(Exception): + pass + + +class TooManyPixels(Exception): + pass + + +# ---------- имя файла и заголовки ---------- + +def validate_filename(value: str | None) -> str: + """filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов.""" + if value is None: + raise bad_request("Query parameter filename is required") + if not 1 <= len(value) <= 255 or not value.strip(): + raise bad_request("filename must be 1-255 characters") + if value in (".", ".."): + raise bad_request("filename is not allowed") + for char in value: + if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"): + raise bad_request("filename contains forbidden characters") + return value + + +def declared_type(header: str | None) -> str: + base = (header or "").split(";", 1)[0].strip().lower() + if base not in ALLOWED_TYPES: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return base + + +def declared_length(header: str | None) -> int | None: + if header is None: + return None + if not header.isdigit(): + raise bad_request("Invalid Content-Length") + length = int(header) + if length > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + return length + + +# ---------- приём тела ---------- + +async def receive(request: Request) -> Upload: + """Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется.""" + spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR) + digest = hashlib.sha256() + size = 0 + deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS + stream = request.stream().__aiter__() + try: + while True: + wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic()) + if wait <= 0: + raise bad_request("Upload took too long") + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except StopAsyncIteration: + break + except asyncio.TimeoutError: + raise bad_request("Upload stalled or took too long") from None + if not chunk: + continue + size += len(chunk) + if size > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + digest.update(chunk) + spool.write(chunk) + except ClientDisconnect: + spool.close() + log.info("Upload aborted by the client after %s bytes, nothing stored", size) + raise bad_request("Upload was interrupted") from None + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise bad_request("Empty body") + return Upload(spool, size, digest.hexdigest()) + + +async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None: + """Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв. + + Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается. + """ + read = 0 + deadline = time.monotonic() + seconds + try: + stream = request.stream().__aiter__() + while read <= limit: + wait = deadline - time.monotonic() + if wait <= 0: + return + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except (StopAsyncIteration, asyncio.TimeoutError): + return + read += len(chunk) + except (ClientDisconnect, RuntimeError): + return + + +# ---------- проверка содержимого ---------- + +def sniff(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return PNG + if head.startswith(JPEG_SIGNATURE): + return JPEG + return None + + +def inspect(upload: Upload, declared: str) -> ImageInfo: + upload.file.seek(0) + detected = sniff(upload.file.read(16)) + if detected is None: + raise unsupported_media_type("File is not a PNG or JPEG image") + if detected != declared: + raise unsupported_media_type("Content-Type does not match the file content") + upload.file.seek(0) + try: + with _decode_slots: + width, height = check_png(upload.file) if detected == PNG else check_jpeg(upload.file) + except TooManyPixels: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except Damaged as exc: + log.info("Rejected damaged %s: %s", detected, exc) + raise unsupported_media_type("Image is damaged or not supported") from None + except Image.DecompressionBombError: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc: + log.info("Rejected damaged %s: %s", detected, type(exc).__name__) + raise unsupported_media_type("Image is damaged or not supported") from None + finally: + upload.file.seek(0) + return ImageInfo(detected, width, height) + + +def check_pixels(width: int, height: int) -> None: + if width < 1 or height < 1: + raise Damaged("zero dimension") + if width * height > settings.MAX_IMAGE_PIXELS: + raise TooManyPixels() + + +def check_jpeg(file: BinaryIO) -> tuple[int, int]: + with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS + width, height = image.size + check_pixels(width, height) + # Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8: + # 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB. + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + return width, height + + +# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR). +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +FILTER_BYTES = bytes(range(5)) + + +class PngRows: + """Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки.""" + + def __init__(self, width, height, depth, color, interlace): + bits = depth * PNG_CHANNELS[color] + + def row_bytes(w): + return (w * bits + 7) // 8 + + passes = [(width, height)] if interlace == 0 else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + # (длина строки с байтом фильтра, байт в проходе) + self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0] + self.expected = sum(total for _, total in self.passes) + self.index, self.pos = 0, 0 + self.total = 0 + + def feed(self, data: bytes) -> None: + view = memoryview(data) + self.total += len(data) + if self.total > self.expected: + raise Damaged("more pixel data than the header says") + offset = 0 + while offset < len(view): + stride, size = self.passes[self.index] + take = min(len(view) - offset, size - self.pos) + first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride + if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES): + raise Damaged("invalid filter type") + self.pos += take + offset += take + if self.pos == size: + self.index, self.pos = self.index + 1, 0 + + def done(self) -> bool: + return self.total == self.expected + + +def check_png(file: BinaryIO) -> tuple[int, int]: + if file.read(8) != PNG_SIGNATURE: + raise Damaged("signature") + rows = inflater = None + width = height = 0 + seen_idat = idat_finished = have_plte = False + color = None + first = True + while True: + header = file.read(8) + if len(header) < 8: + raise Damaged("truncated: no IEND") + length, kind = struct.unpack(">I4s", header) + if length > 0x7FFFFFFF: + raise Damaged("chunk length") + if first and kind != b"IHDR": + raise Damaged("IHDR must be first") + crc = zlib.crc32(kind) + keep = kind in (b"IHDR",) + kept = b"" + left = length + while left: + piece = file.read(min(CRC_CHUNK, left)) + if not piece: + raise Damaged("truncated chunk") + left -= len(piece) + crc = zlib.crc32(piece, crc) + if keep: + kept += piece + elif kind == b"IDAT": + if rows is None or idat_finished: + raise Damaged("IDAT out of order") + feed_inflate(inflater, rows, piece) + stored = file.read(4) + if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: + raise Damaged(f"CRC mismatch in {kind!r}") + + if kind == b"IHDR": + if not first or length != 13: + raise Damaged("IHDR") + width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) + if width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise Damaged("dimensions") + check_pixels(width, height) # до распаковки + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]: + raise Damaged("color type / bit depth") + if compression != 0 or filter_method != 0 or interlace not in (0, 1): + raise Damaged("IHDR methods") + rows = PngRows(width, height, depth, color, interlace) + inflater = zlib.decompressobj() + elif kind == b"PLTE": + have_plte = True + elif kind == b"IDAT": + seen_idat = True + elif kind == b"IEND": + break + if seen_idat and kind != b"IDAT": + idat_finished = True + first = False + + if file.read(1): + raise Damaged("data after IEND") + if color == 3 and not have_plte: + raise Damaged("palette image without PLTE") + if not seen_idat: + raise Damaged("no IDAT") + tail = inflater.flush(INFLATE_CHUNK) + if tail: + rows.feed(tail) + if not inflater.eof or inflater.unused_data or not rows.done(): + raise Damaged("pixel data is incomplete") + return width, height + + +def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: + """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" + if inflater.eof: + if data: + raise Damaged("data after the end of zlib stream") + return + out = inflater.decompress(data, INFLATE_CHUNK) + rows.feed(out) + while inflater.unconsumed_tail and not inflater.eof: + out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK) + rows.feed(out) + if inflater.eof and inflater.unused_data: + raise Damaged("data after the end of zlib stream") -- 2.54.0 From 4d4b914676bde47e65784c987d83c26e981079e2 Mon Sep 17 00:00:00 2001 From: kalibr456 <145665723+kalibr456@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:30:53 +0300 Subject: [PATCH 2/2] =?UTF-8?q?=204:=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BF=D0=BE=20=D1=80?= =?UTF-8?q?=D0=B5=D0=B2=D1=8C=D1=8E=20=E2=80=94=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=BA=D0=B0=20=D1=81=D0=BC=D1=8B=D1=81=D0=BB=D0=B0?= =?UTF-8?q?=20=D1=87=D0=B0=D0=BD=D0=BA=D0=BE=D0=B2=20PNG=20(=D0=BF=D0=B0?= =?UTF-8?q?=D0=BB=D0=B8=D1=82=D1=80=D0=B0,=20tRNS,=20=D0=BF=D0=BE=D1=80?= =?UTF-8?q?=D1=8F=D0=B4=D0=BE=D0=BA,=20=D1=82=D0=B5=D0=BA=D1=81=D1=82),=20?= =?UTF-8?q?=D1=80=D0=B5=D0=B3=D1=80=D0=B5=D1=81=D1=81=D0=B8=D0=BE=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=B5=20=D1=82=D0=B5=D1=81=D1=82=D1=8B,=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 31 ++--- evidence/attachments.txt | 37 +++++- evidence/balancing.txt | 16 +-- evidence/check.txt | 2 +- evidence/consistency.txt | 36 +++--- evidence/db.txt | 8 +- evidence/explain-history.txt | 87 ++++++------- evidence/explain.txt | 43 +++---- evidence/failover.txt | 34 ++--- evidence/failures.txt | 8 +- evidence/idempotency.txt | 6 +- evidence/load.json | 6 +- evidence/load.txt | 10 +- evidence/persistence.txt | 34 ++--- evidence/race.txt | 6 +- evidence/redis.txt | 8 +- evidence/s3-security.txt | 4 +- evidence/smoke.txt | 4 +- evidence/stack.txt | 28 ++--- evidence/student-tests.txt | 4 +- evidence/summary.md | 47 ++++--- evidence/upload-memory.json | 102 +++++++-------- evidence/upload-memory.txt | 12 +- student-tests/images.py | 25 +++- student-tests/test_attachments.py | 70 ++++++++++- uploads.py | 198 +++++++++++++++++++++++++++--- 26 files changed, 587 insertions(+), 279 deletions(-) diff --git a/REPORT.md b/REPORT.md index 527b32b8..f1dbf763 100644 --- a/REPORT.md +++ b/REPORT.md @@ -91,7 +91,7 @@ s3-init: одноразово создаёт приватный bucket и пр 3. Тип по сигнатуре (`\x89PNG...`, `FF D8 FF`), не по заголовку. Не PNG/JPEG -> 415; заявленный тип не совпадает с содержимым -> 415. 4. Размер в пикселях читается из заголовка (IHDR / SOF) и проверяется ДО распаковки: больше 40 000 000 -> 413. 5. Проверка целостности с ограниченной памятью: - - PNG — своя потоковая проверка (`uploads.check_png`): CRC каждого чанка, порядок IHDR/PLTE/IDAT/IEND, допустимые тип цвета и глубина, ничего после IEND; IDAT распаковывается zlib кусками по 64 KiB (вывод не хранится), считается точный объём данных по IHDR (с учётом Adam7) и проверяется байт фильтра каждой строки. «Zip-бомба» не раздувает память: распаковка идёт порциями и обрывается, как только данных больше, чем обещал заголовок. + - PNG — своя потоковая проверка (`uploads.check_png`): CRC каждого чанка и смысл критичных чанков по спецификации PNG (размер палитры 1–256 записей и не больше 2^глубина, размер tRNS по типу цвета, пустой IEND, порядок чанков: всё, что обязано идти до IDAT, — до него, IDAT подряд, неизвестный критичный чанк запрещён); IDAT распаковывается кусками по 64 KiB с подсчётом точного объёма и проверкой байта фильтра каждой строки. Затем Pillow разбирает метаданные до IDAT (Image.open, без verify: verify читает каждый чанк, включая IDAT до 10 MiB, в память). После IDAT допускаются только tIME, tEXt, zTXt, iTXt; текстовые чанки проверяются потоково (ключ 1–79 байт, корректный zlib, не больше 1 MiB распакованного текста); анимированные PNG (acTL/fcTL/fdAT) отклоняются; чанк метаданных до IDAT — не больше 2 MiB. Исправлено по замечанию: раньше проверялись только CRC, и PNG с палитрой из 257 записей получал 201. - JPEG — Pillow декодирует весь поток, но в масштабе 1/8 (`draft`): 40 Мпикс -> ~0,6 Мпикс, около 2 MiB. - Одновременно в реплике проверяется не больше `DECODE_CONCURRENCY=2` файлов (CPU). Битый файл -> 415. Любая ошибка разбора ловится и превращается в 415, а не 500. Проверка PNG сверена с Pillow: 400 случайных корректных PNG разных типов приняты, 400 испорченных (обрезка, перевёрнутый бит) отклонены. @@ -187,36 +187,36 @@ S3-клиент: connect timeout 1 с, read timeout 5 с, всего 2 попы Методика памяти (`student-tests/upload_memory.py`): три сценария — 10 x PNG из случайного шума ~9,5 MiB (почти несжимаемый, максимум трафика); 10 x PNG 8000x5000 RGB (40 Мпикс, максимум распаковки); 20 смешанных. Перед каждым сценарием `docker compose restart api` (сбрасывает пик), прогрев 8 загрузками, затем все потоки стартуют одновременно (barrier) через балансировщик. Пик RSS — `VmHWM` из `/proc/1/status` процесса API (максимум за жизнь процесса по данным ядра, короткие всплески не теряются), дополнительно пик памяти cgroup контейнера и опрос раз в ~0,1–0,5 с. Проверяются коды 201 и SHA-256 скачанных файлов. Клиент — тот же компьютер, поэтому время загрузки ограничено и клиентом; это замер памяти, а не пропускной способности. -Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-22, commit c942c61 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): | Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | | --- | --- | --- | --- | --- | --- | -| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 | -| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 | +| 1 | 343119df2488 | 2 | 123 | 0 | 4.422 | +| 2 | d590adc21943 | 2 | 124 | 0 | 4.531 | Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: | Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 | -| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 | -| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 | -| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 | -| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 | -| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 2.87 / 3.52 | 343119df2488: 5 | 94.8 | 107.6 | 12.8 | 136.0 | +| | | | | d590adc21943: 5 | 91.2 | 104.5 | 13.3 | 136.2 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 343119df2488: 5 | 95.1 | 96.6 | 1.5 | 86.5 | +| | | | | d590adc21943: 5 | 92.1 | 96.6 | 4.6 | 83.5 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.5 / 1.78 | 343119df2488: 10 | 93.2 | 116.2 | 23.0 | 158.2 | +| | | | | d590adc21943: 10 | 95.0 | 118.6 | 23.6 | 167.8 | Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. -Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.325 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет | -| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет | -| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | -| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.422, 4.531 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 47.0 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19453 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14891 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | | падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | | обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | @@ -231,5 +231,6 @@ S3-клиент: connect timeout 1 с, read timeout 5 с, всего 2 попы - Обход bucket janitor’ом держит в памяти список ключей (для учебного объёма — мегабайты); для миллионов объектов нужна сверка по диапазонам ключей или S3 Inventory. - Временный файл загрузки (часть тела сверх 256 KiB) лежит на диске контейнера. В Linux `tempfile.TemporaryFile` создаётся уже без имени в каталоге (O_TMPFILE), поэтому даже при SIGKILL реплики место освобождается ядром, «забытых» файлов не бывает. Одновременно на диске может быть до 10 MiB на каждую идущую загрузку; лимита на число одновременных загрузок, кроме лимитов HAProxy/uvicorn, нет. - Проверяется структура изображения, а не его безопасность для всех декодеров (например, JPEG с некорректными, но декодируемыми данными пройдёт). Перекодирование оригинала — задача lab5. +- PNG с частными служебными чанками после IDAT и анимированные PNG отклоняются (строже спецификации, на практике не встречаются в фотографиях). - MinIO собирался быть первым выбором, но его образы больше не доступны в публичных реестрах: закреплённый тег защищает от неожиданного обновления, но не от исчезновения образа. Для эксплуатации нужно зеркалить образы в свой реестр. SeaweedFS запущен одним процессом `weed server` на одном диске — это SPOF и не защита от потери диска; веб-интерфейсы master/filer без авторизации, закрыты только сетью. - Скрипты `failover_metrics.py`, `rolling_update.py`, `test_expiry.py` из lab3 остались в `student-tests/` и работают со стендом, но в общий прогон lab4 не входят (это уровень 4 lab3, а не lab4). diff --git a/evidence/attachments.txt b/evidence/attachments.txt index dc0ed88a..b9820b55 100644 --- a/evidence/attachments.txt +++ b/evidence/attachments.txt @@ -1,7 +1,7 @@ # attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids -# date: 2026-10-21 21:03:05, commit: 2dbed80 +# date: 2026-10-22 14:19:26, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_attachments.py -# exit code: 0, 4.0 s +# exit code: 0, 5.4 s sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes @@ -14,7 +14,7 @@ exactly 10485760 bytes (10 MiB) -> 201 PASS: the 10 MiB limit is exact 12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed PASS: oversized streamed body is cut off, the API keeps working -8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 201 5ffe6847 +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 201 af872a80 8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded PASS: pixel limit is checked before decompression @@ -30,6 +30,35 @@ random bytes (1600 bytes, image/png) -> 415 plain text (28 bytes, image/jpeg) -> 415 empty body -> 400 PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted PNG bytes sent as image/jpeg -> 415 JPEG bytes sent as image/png -> 415 GIF bytes sent as image/png -> 415 @@ -64,4 +93,4 @@ PASS: Idempotency-Key covers attachment_ids sent 1.1 MB of a declared 5 MB body and closed the connection the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) PASS: an interrupted upload is dropped -11 passed, 0 failed +12 passed, 0 failed diff --git a/evidence/balancing.txt b/evidence/balancing.txt index dbbf431c..60381951 100644 --- a/evidence/balancing.txt +++ b/evidence/balancing.txt @@ -1,18 +1,18 @@ # balancing: balancing, shared sessions/data/files, X-Instance-Id -# date: 2026-10-21 21:08:50, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:27, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_balancing.py -# exit code: 0, 1.9 s +# exit code: 0, 2.0 s -20 x GET /users/me with one token -> replicas: {'6128eb268c44': 10, '11a93acecaca': 10} +20 x GET /users/me with one token -> replicas: {'d590adc21943': 10, '343119df2488': 10} PASS: one session works on every replica, no second login (sessions are shared in Redis) -message written by 11a93acecaca; read back by ['11a93acecaca', '6128eb268c44'] +message written by 343119df2488; read back by ['343119df2488', 'd590adc21943'] PASS: a message written via one replica is read right away via another (shared PostgreSQL) -logout on 6128eb268c44; old token then rejected by ['11a93acecaca', '6128eb268c44'] +logout on d590adc21943; old token then rejected by ['343119df2488', 'd590adc21943'] PASS: logout on one replica revokes the session on all of them -first POST on 6128eb268c44 -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af; same key on 11a93acecaca -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af +first POST on d590adc21943 -> 201 6c2f624d-c728-4738-b486-b17ab735c0ae; same key on 343119df2488 -> 201 6c2f624d-c728-4738-b486-b17ab735c0ae PASS: Idempotency-Key is shared: a retry through another replica returns the same message -file uploaded via 6128eb268c44; downloaded byte for byte via ['11a93acecaca', '6128eb268c44'] +file uploaded via d590adc21943; downloaded byte for byte via ['343119df2488', 'd590adc21943'] PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local) -statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['11a93acecaca', '6128eb268c44']; api containers: ['11a93acecaca', '6128eb268c44'] +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['343119df2488', 'd590adc21943']; api containers: ['343119df2488', 'd590adc21943'] PASS: X-Instance-Id is in every response and points to a real container 6 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt index 61c52c64..8d746957 100644 --- a/evidence/check.txt +++ b/evidence/check.txt @@ -1,5 +1,5 @@ # check: offline contract check (make check) -# date: 2026-10-21 21:02:57, commit: 2dbed80 +# date: 2026-10-22 14:19:16, commit: c942c61 + uncommitted changes # command: python.exe scripts/check_contract.py # exit code: 0, 0.1 s diff --git a/evidence/consistency.txt b/evidence/consistency.txt index b345844f..e4cb4769 100644 --- a/evidence/consistency.txt +++ b/evidence/consistency.txt @@ -1,47 +1,47 @@ # consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage -# date: 2026-10-21 21:08:14, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:21:48, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_consistency.py -# exit code: 0, 148.7 s +# exit code: 0, 133.0 s declared 9 000 000 bytes, sent 211817, closed the connection declared 9 000 000 bytes, sent 3111817, closed the connection -rows in attachments: 32 -> 32; objects in bucket: 31 -> 31 -temporary upload files in api containers: {'11a93acecaca': 0, '6128eb268c44': 0} (removed when the request ends) +rows in attachments: 448 -> 448; objects in bucket: 444 -> 444 +temporary upload files in api containers: {'343119df2488': 0, 'd590adc21943': 0} (removed when the request ends) api containers have no volumes: the original is never stored locally PASS: an interrupted upload leaves no row, no object and no local file -crash after S3 PUT: row 04aacbf6 state=uploading, object originals/219175ec1d5647f0aeaec84ee8243cb3 exists in S3 +crash after S3 PUT: row 9aa0bc8a state=uploading, object originals/8ae03455b55a4f798b7bc6248b450ef9 exists in S3 the half-written attachment is invisible: GET -> 404, attach to a message -> 404 -janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.101} -> fresh row and object are kept (the upload might still be running) -row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.092} -> nothing deleted -janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.096} -> row and object deleted -janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.103} -> idempotent +janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 446, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.133} -> fresh row and object are kept (the upload might still be running) +row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "objects_scanned": 446, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.184} -> nothing deleted +janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 445, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.132} -> row and object deleted +janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 445, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.179} -> idempotent the ready attachment of the same chat is untouched PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period -object originals/867f5c5d42bf4bf9973b60ef8856d88f put into the bucket without a row in PostgreSQL -janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.102} -> fresh object kept -after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.098} -> reported, kept -janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.1} -> orphan deleted +object originals/101748e3a911439a9c1e11e768c99822 put into the bucket without a row in PostgreSQL +janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 447, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.191} -> fresh object kept +after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 447, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.188} -> reported, kept +janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 447, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 4, "seconds": 0.1} -> orphan deleted ready attachments older than the grace period are untouched (they have rows) -object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500) +object of a ready attachment deleted behind the API's back: janitor reports missing_objects=5 and keeps the row; download -> 503 (not 500) PASS: orphans are found and deleted only after the grace period; missing objects are reported --- S3 stopped (connection refused / name not resolved): docker compose stop s3 -upload -> 503 unavailable in 20141 ms, no storage details in the body +upload -> 503 unavailable in 19453 ms, no storage details in the body download of an existing file -> 503 metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API docker compose start s3 after S3 recovery the old file downloads byte for byte new upload -> 201 -janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 35, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.099} +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 448, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 5, "seconds": 0.191} PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up --- S3 frozen (no answer, timeouts): docker compose pause s3 -upload -> 503 unavailable in 14766 ms, no storage details in the body +upload -> 503 unavailable in 14891 ms, no storage details in the body download of an existing file -> 503 metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API docker compose unpause s3 after S3 recovery the old file downloads byte for byte new upload -> 201 -janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 37, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.094} +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 450, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 5, "seconds": 0.115} PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up 5 passed, 0 failed diff --git a/evidence/db.txt b/evidence/db.txt index 481e4035..77cb2e79 100644 --- a/evidence/db.txt +++ b/evidence/db.txt @@ -1,9 +1,9 @@ # db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations -# date: 2026-10-21 21:09:10, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:40, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_db.py -# exit code: 0, 14.2 s +# exit code: 0, 6.2 s -chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 57 -> 58 +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 335 -> 336 PASS: chat row was inserted and then rolled back together with members direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" PASS: the database itself rejects a message from a non-member @@ -14,7 +14,7 @@ PASS: username is unique in the database applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; running the migrate service once more after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; log: Migrations applied: none (schema is up to date) PASS: migrations are applied once, a repeated run does nothing -api replicas that checked the schema at start: 8; api log lines 'Migrations applied': 0; migrate service: Exited (0) 6 minutes ago +api replicas that checked the schema at start: 14; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago PASS: only the migrate service applies migrations, replicas wait for the schema unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt index 678f4735..46f4949a 100644 --- a/evidence/explain-history.txt +++ b/evidence/explain-history.txt @@ -1,7 +1,7 @@ === Data volume messages_total | messages_in_big_chat ----------------+---------------------- - 20146 | 15000 + 35618 | 15000 === History index @@ -13,39 +13,39 @@ === 1. First page: limit=50, the app asks for 51 rows to know if there is a next page QUERY PLAN ------------------------------------------------------------------------------------------------ - Limit (actual time=0.005..0.011 rows=51 loops=1) - Buffers: shared hit=5 + Limit (actual time=0.006..0.010 rows=51 loops=1) + Buffers: shared hit=4 -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) - Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) - Buffers: shared hit=5 + Index Cond: (chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) + Buffers: shared hit=4 Planning: - Buffers: shared hit=6 - Planning Time: 0.060 ms + Buffers: shared hit=8 + Planning Time: 0.071 ms Execution Time: 0.020 ms === 2. Page after the 10000th message by keyset cursor (this is what the app does) QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Limit (actual time=0.005..0.012 rows=51 loops=1) - Buffers: shared hit=6 - -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) - Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) - Buffers: shared hit=6 - Planning Time: 0.044 ms - Execution Time: 0.031 ms + Limit (actual time=0.006..0.011 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) + Index Cond: ((chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '4bd22867-547f-4ecf-8182-443fbb004188'::uuid))) + Buffers: shared hit=5 + Planning Time: 0.040 ms + Execution Time: 0.029 ms === 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Limit (actual time=0.020..0.225 rows=51 loops=1) - Buffers: shared hit=60 - -> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1) - Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) - Buffers: shared hit=60 + Limit (actual time=0.019..0.208 rows=51 loops=1) + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.019..0.204 rows=51 loops=1) + Index Cond: ((chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '4bd22867-547f-4ecf-8182-443fbb004188'::uuid))) + Buffers: shared hit=59 SubPlan 1 - -> Sort (actual time=0.003..0.003 rows=0 loops=51) + -> Sort (actual time=0.002..0.002 rows=0 loops=51) Sort Key: a."position" Sort Method: quicksort Memory: 25kB Buffers: shared hit=54 @@ -57,39 +57,44 @@ Buffers: shared hit=51 Planning: Buffers: shared hit=63 - Planning Time: 0.157 ms - Execution Time: 0.242 ms + Planning Time: 0.156 ms + Execution Time: 0.225 ms === 3. The same page with OFFSET 10000 (the app does NOT do this) - QUERY PLAN ---------------------------------------------------------------------------------------------------- - Limit (actual time=0.878..0.884 rows=51 loops=1) - Buffers: shared hit=258 - -> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1) - Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) - Buffers: shared hit=258 - Planning Time: 0.037 ms - Execution Time: 0.897 ms + QUERY PLAN +-------------------------------------------------------------------------------- + Limit (actual time=3.056..3.060 rows=51 loops=1) + Buffers: shared hit=585 + -> Sort (actual time=2.647..2.857 rows=10051 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 2377kB + Buffers: shared hit=585 + -> Seq Scan on messages (actual time=0.005..1.278 rows=15000 loops=1) + Filter: (chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) + Rows Removed by Filter: 20618 + Buffers: shared hit=585 + Planning Time: 0.033 ms + Execution Time: 3.107 ms === 4. Query 2 without the index (index is dropped only inside this transaction) QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Limit (actual time=1.097..1.101 rows=51 loops=1) - Buffers: shared hit=261 - -> Sort (actual time=1.097..1.098 rows=51 loops=1) + Limit (actual time=1.432..1.436 rows=51 loops=1) + Buffers: shared hit=585 + -> Sort (actual time=1.432..1.433 rows=51 loops=1) Sort Key: created_at, id Sort Method: top-N heapsort Memory: 31kB - Buffers: shared hit=261 - -> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1) - Filter: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid))) - Rows Removed by Filter: 15146 - Buffers: shared hit=261 + Buffers: shared hit=585 + -> Seq Scan on messages (actual time=0.678..1.116 rows=5000 loops=1) + Filter: ((chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '4bd22867-547f-4ecf-8182-443fbb004188'::uuid))) + Rows Removed by Filter: 30618 + Buffers: shared hit=585 Planning: Buffers: shared hit=4 - Planning Time: 0.051 ms - Execution Time: 1.115 ms + Planning Time: 0.049 ms + Execution Time: 1.450 ms === ROLLBACK: test data removed, index is back diff --git a/evidence/explain.txt b/evidence/explain.txt index 597939b2..ac52e43f 100644 --- a/evidence/explain.txt +++ b/evidence/explain.txt @@ -1,43 +1,44 @@ # explain: EXPLAIN of the history page (full plans: explain-history.txt) -# date: 2026-10-21 21:09:14, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:45, commit: c942c61 + uncommitted changes # command: python.exe student-tests/explain.py # exit code: 0, 1.1 s === Data volume === History index === 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=4 + -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) + Buffers: shared hit=4 + Buffers: shared hit=8 + Execution Time: 0.020 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) Buffers: shared hit=5 -> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1) Buffers: shared hit=5 - Buffers: shared hit=6 - Execution Time: 0.020 ms -=== 2. Page after the 10000th message by keyset cursor (this is what the app does) - Buffers: shared hit=6 - -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) - Buffers: shared hit=6 - Execution Time: 0.031 ms + Execution Time: 0.029 ms === 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message - Buffers: shared hit=60 - -> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1) - Buffers: shared hit=60 + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.019..0.204 rows=51 loops=1) + Buffers: shared hit=59 Buffers: shared hit=54 -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) Buffers: shared hit=51 -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) Buffers: shared hit=51 Buffers: shared hit=63 - Execution Time: 0.242 ms + Execution Time: 0.225 ms === 3. The same page with OFFSET 10000 (the app does NOT do this) - Buffers: shared hit=258 - -> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1) - Buffers: shared hit=258 - Execution Time: 0.897 ms + Buffers: shared hit=585 + Buffers: shared hit=585 + -> Seq Scan on messages (actual time=0.005..1.278 rows=15000 loops=1) + Buffers: shared hit=585 + Execution Time: 3.107 ms === 4. Query 2 without the index (index is dropped only inside this transaction) - Buffers: shared hit=261 - Buffers: shared hit=261 - -> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1) - Buffers: shared hit=261 + Buffers: shared hit=585 + Buffers: shared hit=585 + -> Seq Scan on messages (actual time=0.678..1.116 rows=5000 loops=1) + Buffers: shared hit=585 Buffers: shared hit=4 - Execution Time: 1.115 ms + Execution Time: 1.450 ms === ROLLBACK: test data removed, index is back PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover.txt b/evidence/failover.txt index 9bb22e80..26db5df4 100644 --- a/evidence/failover.txt +++ b/evidence/failover.txt @@ -1,33 +1,33 @@ # failover: public failover test for every replica, file downloaded via the other one (make test-failover) -# date: 2026-10-21 21:09:54, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:23:28, commit: c942c61 + uncommitted changes # command: python.exe student-tests/run_failover.py -# exit code: 0, 39.7 s +# exit code: 0, 43.2 s -hook: scripts\stop-one.cmd; api replicas: ['11a93acecaca', '6128eb268c44'] +hook: scripts\stop-one.cmd; api replicas: ['343119df2488', 'd590adc21943'] === run 1: api containers before SERVICE CONTAINER ID STATUS -api 11a93acecaca Up 36 seconds (healthy) -api 6128eb268c44 Up 36 seconds (healthy) +api 343119df2488 Up 27 seconds (healthy) +api d590adc21943 Up 27 seconds (healthy) Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.39} +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.422} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. api containers after SERVICE CONTAINER ID STATUS -api 11a93acecaca Exited (0) 11 seconds ago -api 6128eb268c44 Up 53 seconds (healthy) -the hook stopped 11a93acecaca (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched -restored: docker start 11a93acecaca -> healthy and back in HAProxy +api 343119df2488 Exited (0) 11 seconds ago +api d590adc21943 Up 43 seconds (healthy) +the hook stopped 343119df2488 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 343119df2488 -> healthy and back in HAProxy === run 2: api containers before SERVICE CONTAINER ID STATUS -api 11a93acecaca Up 2 seconds (healthy) -api 6128eb268c44 Up 56 seconds (healthy) +api 343119df2488 Up 4 seconds (healthy) +api d590adc21943 Up 48 seconds (healthy) Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 4.328} +{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 4.531} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. api containers after SERVICE CONTAINER ID STATUS -api 11a93acecaca Up 18 seconds (healthy) -api 6128eb268c44 Exited (0) 11 seconds ago -the hook stopped 6128eb268c44 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched -restored: docker start 6128eb268c44 -> healthy and back in HAProxy +api 343119df2488 Up 20 seconds (healthy) +api d590adc21943 Exited (0) 11 seconds ago +the hook stopped d590adc21943 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start d590adc21943 -> healthy and back in HAProxy PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt index 28d1d694..ce7d206f 100644 --- a/evidence/failures.txt +++ b/evidence/failures.txt @@ -1,11 +1,11 @@ # failures: Redis and PostgreSQL outages via the balancer -# date: 2026-10-21 21:11:20, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:24:54, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_failures.py # exit code: 0, 55.6 s stopping redis -/health/live -> 200, /health/ready -> 503 (8016 ms) -valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked +/health/live -> 200, /health/ready -> 503 (8000 ms) +valid token: GET /users/me -> 503 unavailable (8031 ms), session cannot be checked unknown token -> 503, not 200: nobody is trusted without a check no token -> 401 login -> 503: there is nowhere to store the session @@ -19,7 +19,7 @@ starting redis after start: the session issued before the crash still works (AOF, appendfsync always) PASS: sessions survive a Redis crash, not only a clean restart stopping postgres -/health/live -> 200, /health/ready -> 503 (1016 ms) +/health/live -> 200, /health/ready -> 503 (1031 ms) valid token: users/me, chats, history, new message -> 503 unavailable, no details no token -> 401, unknown token -> 401: anonymous users are not let in login -> 503, registration -> 503 diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt index 205a798d..fce04326 100644 --- a/evidence/idempotency.txt +++ b/evidence/idempotency.txt @@ -1,11 +1,11 @@ # idempotency: Idempotency-Key, parallel same key via replicas -# date: 2026-10-21 21:08:52, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:30, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_idempotency.py -# exit code: 0, 2.4 s +# exit code: 0, 2.5 s first -> 201; same JSON written differently -> 201; plain retry -> 201 PASS: a retry with the same key returns the original 201 and message, no duplicate -20 parallel POSTs with one key: statuses {201: 20}, replicas {'6128eb268c44': 10, '11a93acecaca': 10}, different bodies in answers: 1, messages in history: 1 +20 parallel POSTs with one key: statuses {201: 20}, replicas {'343119df2488': 10, 'd590adc21943': 10}, different bodies in answers: 1, messages in history: 1 PASS: parallel requests with one key through different replicas create exactly one message 20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A'] PASS: the first body wins, the same body gets 201, the other body gets 409 diff --git a/evidence/load.json b/evidence/load.json index c4db3afc..c41ff897 100644 --- a/evidence/load.json +++ b/evidence/load.json @@ -2,12 +2,12 @@ "lab": 4, "workload": "closed-loop POST messages; no automatic retries", "requested_duration_seconds": 30.0, - "elapsed_seconds": 30.015, + "elapsed_seconds": 30.047, "concurrency": 4, - "successful_requests": 7379, + "successful_requests": 7191, "failed_requests": 0, "error_ratio": 0.0, - "successful_rps": 245.844, + "successful_rps": 239.325, "successful_latency_ms": { "p50": 16.0, "p95": 32.0, diff --git a/evidence/load.txt b/evidence/load.txt index 8529586e..2d10d4a0 100644 --- a/evidence/load.txt +++ b/evidence/load.txt @@ -1,18 +1,18 @@ # load: public load generator, 30 s, concurrency 4 -# date: 2026-10-21 21:10:24, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:23:58, commit: c942c61 + uncommitted changes # command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json -# exit code: 0, 30.4 s +# exit code: 0, 30.3 s { "lab": 4, "workload": "closed-loop POST messages; no automatic retries", "requested_duration_seconds": 30.0, - "elapsed_seconds": 30.015, + "elapsed_seconds": 30.047, "concurrency": 4, - "successful_requests": 7379, + "successful_requests": 7191, "failed_requests": 0, "error_ratio": 0.0, - "successful_rps": 245.844, + "successful_rps": 239.325, "successful_latency_ms": { "p50": 16.0, "p95": 32.0, diff --git a/evidence/persistence.txt b/evidence/persistence.txt index f23f416b..b7334d58 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,24 +1,24 @@ # persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence) -# date: 2026-10-21 21:12:49, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:25:42, commit: c942c61 + uncommitted changes # command: python.exe student-tests/run_persistence.py -# exit code: 0, 40.2 s +# exit code: 0, 47.9 s before: container, ID, started at - messenger-lab4-api-1 11a93acecaca 2026-10-21T18:11:37.067348364Z - messenger-lab4-api-2 6128eb268c44 2026-10-21T18:11:36.916459116Z - messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:11:33.313531682Z - messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:11:40.981519931Z - messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:11:22.482229396Z - messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:11:22.626999402Z - messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:11:32.264154477Z + messenger-lab4-api-1 343119df2488 2026-10-22T11:23:03.387600262Z + messenger-lab4-api-2 d590adc21943 2026-10-22T11:23:24.689368662Z + messenger-lab4-janitor-1 d50ac08018c1 2026-10-22T11:18:17.423165415Z + messenger-lab4-lb-1 0d5b262b09b3 2026-10-22T11:03:30.971491955Z + messenger-lab4-postgres-1 84794c3ac2c5 2026-10-22T11:24:52.672662576Z + messenger-lab4-redis-1 3130fd5532b0 2026-10-22T11:24:32.055032931Z + messenger-lab4-s3-1 122ad603e3be 2026-10-22T11:20:52.20174809Z Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. -after (hook + waiting for readiness took 39.2 s): container, ID, started at, restarted? - messenger-lab4-api-1 11a93acecaca 2026-10-21T18:12:46.288933033Z same container: True, restarted: True - messenger-lab4-api-2 6128eb268c44 2026-10-21T18:12:46.174283308Z same container: True, restarted: True - messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:12:42.503993973Z same container: True, restarted: True - messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:12:47.684809415Z same container: True, restarted: True - messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:12:11.527270274Z same container: True, restarted: True - messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:12:11.368416107Z same container: True, restarted: True - messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:12:21.264135454Z same container: True, restarted: True +after (hook + waiting for readiness took 47.0 s): container, ID, started at, restarted? + messenger-lab4-api-1 343119df2488 2026-10-22T11:25:36.201352227Z same container: True, restarted: True + messenger-lab4-api-2 d590adc21943 2026-10-22T11:25:36.073058122Z same container: True, restarted: True + messenger-lab4-janitor-1 d50ac08018c1 2026-10-22T11:25:32.252887377Z same container: True, restarted: True + messenger-lab4-lb-1 0d5b262b09b3 2026-10-22T11:25:40.174401907Z same container: True, restarted: True + messenger-lab4-postgres-1 84794c3ac2c5 2026-10-22T11:24:56.311470336Z same container: True, restarted: True + messenger-lab4-redis-1 3130fd5532b0 2026-10-22T11:24:56.150164333Z same container: True, restarted: True + messenger-lab4-s3-1 122ad603e3be 2026-10-22T11:25:06.0234901Z same container: True, restarted: True PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, new start time); data, the attachment bytes and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt index 7e5c4dfd..b638d877 100644 --- a/evidence/race.txt +++ b/evidence/race.txt @@ -1,9 +1,9 @@ # race: lab2 races via the balancer -# date: 2026-10-21 21:08:56, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:34, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_race.py -# exit code: 0, 1.2 s +# exit code: 0, 1.1 s -sent 100 messages from 10 clients in 0.22 s; read 100 in 4 pages, unique ids: 100 +sent 100 messages from 10 clients in 0.19 s; read 100 in 4 pages, unique ids: 100 PASS: 100 concurrent messages from 10 clients, no losses and no duplicates 20 parallel registrations of one username: 201 x1, 409 x19 PASS: concurrent registration created exactly one user diff --git a/evidence/redis.txt b/evidence/redis.txt index b118ac4e..c3f01590 100644 --- a/evidence/redis.txt +++ b/evidence/redis.txt @@ -1,11 +1,11 @@ # redis: Redis as the session store -# date: 2026-10-21 21:09:13, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:43, commit: c942c61 + uncommitted changes # command: python.exe student-tests/redis_info.py -# exit code: 0, 2.8 s +# exit code: 0, 2.9 s -session key: session:f4f1fe0235d139de...; value is the user id: True +session key: session:e57bdd083b4cb87e...; value is the user id: True TTL of the key, seconds: 3599 -sessions in Redis: 126 +sessions in Redis: 330 config appendonly = yes config appendfsync = always config maxmemory-policy = noeviction diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt index d118d616..d5099311 100644 --- a/evidence/s3-security.txt +++ b/evidence/s3-security.txt @@ -1,7 +1,7 @@ # s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap -# date: 2026-10-21 21:03:14, commit: 2dbed80 +# date: 2026-10-22 14:19:35, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_s3_security.py -# exit code: 0, 9.5 s +# exit code: 0, 9.6 s anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied anonymous head_object directly to S3 (from the data network) -> 403 diff --git a/evidence/smoke.txt b/evidence/smoke.txt index 95abed91..9da40550 100644 --- a/evidence/smoke.txt +++ b/evidence/smoke.txt @@ -1,6 +1,6 @@ # smoke: public smoke via the balancer (make test), with attachment -# date: 2026-10-21 21:03:01, commit: 2dbed80 +# date: 2026-10-22 14:19:20, commit: c942c61 + uncommitted changes # command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 -# exit code: 0, 0.6 s +# exit code: 0, 0.8 s PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt index 9b6415d5..ef8ddb1d 100644 --- a/evidence/stack.txt +++ b/evidence/stack.txt @@ -1,21 +1,21 @@ # stack: stand layout: replicas, ports, limits, networks, secrets, S3 -# date: 2026-10-21 21:03:00, commit: 2dbed80 +# date: 2026-10-22 14:19:19, commit: c942c61 + uncommitted changes # command: python.exe student-tests/check_stack.py -# exit code: 0, 3.0 s +# exit code: 0, 2.9 s host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop -SERVICE CONTAINER ID STATUS PORTS -api 11a93acecaca Up 20 seconds (healthy) 8080/tcp -api 6128eb268c44 Up 20 seconds (healthy) 8080/tcp -janitor baefc0e97dff Up 20 seconds -lb 03697e5fe34c Up 18 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp -migrate aca090330db4 Exited (0) 22 seconds ago -postgres f7a6a1ce25a7 Up 2 minutes (healthy) -redis b4523b39a0fe Up 2 minutes (healthy) -s3 e7cbe6ddfa4a Up 2 minutes (healthy) -s3-init 60719e674e18 Exited (0) 21 seconds ago +SERVICE CONTAINER ID STATUS PORTS +api 343119df2488 Up 17 seconds (healthy) 8080/tcp +api d590adc21943 Up 18 seconds (healthy) 8080/tcp +janitor d50ac08018c1 Up About a minute +lb 0d5b262b09b3 Up 15 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +migrate 6c3123ccf237 Exited (0) About a minute ago +postgres 84794c3ac2c5 Up 16 minutes (healthy) +redis 3130fd5532b0 Up 16 minutes (healthy) +s3 122ad603e3be Up 16 minutes (healthy) +s3-init 47c2d7d47ed2 Exited (0) About a minute ago container image cpus memory host ports networks messenger-lab4-api-1 messenger-lab4-api 1.00 512 MiB - data,edge @@ -35,6 +35,6 @@ secrets of redis: ['redis_auth'] secrets of s3: ['s3_config'] secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] -through http://127.0.0.1:8080 one session is served by 2 replicas: ['11a93acecaca', '6128eb268c44'] -/health/ready -> 200 from 11a93acecaca +through http://127.0.0.1:8080 one session is served by 2 replicas: ['343119df2488', 'd590adc21943'] +/health/ready -> 200 from d590adc21943 PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor only in the internal data network, S3 admin keys only in s3 and s3-init diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt index 9ac2c0fe..980ab731 100644 --- a/evidence/student-tests.txt +++ b/evidence/student-tests.txt @@ -1,7 +1,7 @@ # student-tests: lab1/lab2 API tests via the balancer -# date: 2026-10-21 21:08:55, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:33, commit: c942c61 + uncommitted changes # command: python.exe student-tests/test_api.py -# exit code: 0, 2.7 s +# exit code: 0, 2.9 s PASS: duplicate username returns 409 PASS: invalid JSON returns 400 diff --git a/evidence/summary.md b/evidence/summary.md index 5df443a4..465f1819 100644 --- a/evidence/summary.md +++ b/evidence/summary.md @@ -2,39 +2,56 @@ | Шаг | Что проверяет | Результат | Время, с | Файл | | --- | --- | --- | --- | --- | -| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 40 | evidence/persistence.txt | +| check | offline contract check (make check) | PASS | 0 | evidence/check.txt | +| stack | stand layout: replicas, ports, limits, networks, secrets, S3 | PASS | 3 | evidence/stack.txt | +| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/smoke.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 5 | evidence/attachments.txt | +| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt | +| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 133 | evidence/consistency.txt | +| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 37 | evidence/upload-memory.txt | +| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt | +| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt | +| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 6 | evidence/db.txt | +| redis | Redis as the session store | PASS | 3 | evidence/redis.txt | +| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt | +| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 43 | evidence/failover.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| failures | Redis and PostgreSQL outages via the balancer | PASS | 56 | evidence/failures.txt | +| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 48 | evidence/persistence.txt | # Замеры -Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-22, commit c942c61 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB. Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): | Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | | --- | --- | --- | --- | --- | --- | -| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 | -| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 | +| 1 | 343119df2488 | 2 | 123 | 0 | 4.422 | +| 2 | d590adc21943 | 2 | 124 | 0 | 4.531 | Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: | Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 | -| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 | -| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 | -| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 | -| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 | -| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 2.87 / 3.52 | 343119df2488: 5 | 94.8 | 107.6 | 12.8 | 136.0 | +| | | | | d590adc21943: 5 | 91.2 | 104.5 | 13.3 | 136.2 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 343119df2488: 5 | 95.1 | 96.6 | 1.5 | 86.5 | +| | | | | d590adc21943: 5 | 92.1 | 96.6 | 4.6 | 83.5 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.5 / 1.78 | 343119df2488: 10 | 93.2 | 116.2 | 23.0 | 158.2 | +| | | | | d590adc21943: 10 | 95.0 | 118.6 | 23.6 | 167.8 | Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. -Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.325 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет | -| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет | -| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | -| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.422, 4.531 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 47.0 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19453 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14891 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | | падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | | обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json index 30501787..c9eed9e3 100644 --- a/evidence/upload-memory.json +++ b/evidence/upload-memory.json @@ -1,5 +1,5 @@ { - "date": "2026-10-21 21:08", + "date": "2026-10-22 14:22", "api_limit_mib": 512, "api_limit_cpus": 1.0, "runs": [ @@ -11,32 +11,32 @@ ], "total_mib": 94.9, "all_201": true, - "wall_seconds": 1.52, + "wall_seconds": 3.52, "latency_s": { - "p50": 1.38, - "max": 1.52 + "p50": 2.87, + "max": 3.52 }, - "sampler_rounds": 9, + "sampler_rounds": 18, "replicas": [ { - "replica": "11a93acecaca", - "uploads_served": 5, - "rss_before_mib": 93.5, - "rss_peak_mib": 106.2, - "rss_growth_mib": 12.6, - "rss_peak_sampled_mib": 106.2, - "cgroup_peak_mib": 134.1, - "cgroup_peak_sampled_mib": 127.7 - }, - { - "replica": "6128eb268c44", + "replica": "343119df2488", "uploads_served": 5, "rss_before_mib": 94.8, - "rss_peak_mib": 105.3, - "rss_growth_mib": 10.5, - "rss_peak_sampled_mib": 105.3, - "cgroup_peak_mib": 135.2, - "cgroup_peak_sampled_mib": 131.4 + "rss_peak_mib": 107.6, + "rss_growth_mib": 12.8, + "rss_peak_sampled_mib": 107.6, + "cgroup_peak_mib": 136.0, + "cgroup_peak_sampled_mib": 134.5 + }, + { + "replica": "d590adc21943", + "uploads_served": 5, + "rss_before_mib": 91.2, + "rss_peak_mib": 104.5, + "rss_growth_mib": 13.3, + "rss_peak_sampled_mib": 104.5, + "cgroup_peak_mib": 136.2, + "cgroup_peak_sampled_mib": 133.3 } ] }, @@ -53,27 +53,27 @@ "p50": 0.85, "max": 1.09 }, - "sampler_rounds": 8, + "sampler_rounds": 12, "replicas": [ { - "replica": "11a93acecaca", + "replica": "343119df2488", "uploads_served": 5, - "rss_before_mib": 93.2, + "rss_before_mib": 95.1, "rss_peak_mib": 96.6, - "rss_growth_mib": 3.4, + "rss_growth_mib": 1.5, "rss_peak_sampled_mib": 96.6, - "cgroup_peak_mib": 84.3, - "cgroup_peak_sampled_mib": 77.3 + "cgroup_peak_mib": 86.5, + "cgroup_peak_sampled_mib": 77.6 }, { - "replica": "6128eb268c44", + "replica": "d590adc21943", "uploads_served": 5, - "rss_before_mib": 93.1, - "rss_peak_mib": 94.7, - "rss_growth_mib": 1.6, - "rss_peak_sampled_mib": 94.6, - "cgroup_peak_mib": 86.8, - "cgroup_peak_sampled_mib": 77.3 + "rss_before_mib": 92.1, + "rss_peak_mib": 96.6, + "rss_growth_mib": 4.6, + "rss_peak_sampled_mib": 96.6, + "cgroup_peak_mib": 83.5, + "cgroup_peak_sampled_mib": 77.8 } ] }, @@ -87,32 +87,32 @@ ], "total_mib": 133.2, "all_201": true, - "wall_seconds": 1.77, + "wall_seconds": 1.78, "latency_s": { - "p50": 1.37, - "max": 1.77 + "p50": 1.5, + "max": 1.78 }, - "sampler_rounds": 10, + "sampler_rounds": 12, "replicas": [ { - "replica": "11a93acecaca", + "replica": "343119df2488", "uploads_served": 10, - "rss_before_mib": 94.8, - "rss_peak_mib": 121.7, - "rss_growth_mib": 26.9, - "rss_peak_sampled_mib": 121.7, - "cgroup_peak_mib": 181.9, - "cgroup_peak_sampled_mib": 180.6 + "rss_before_mib": 93.2, + "rss_peak_mib": 116.2, + "rss_growth_mib": 23.0, + "rss_peak_sampled_mib": 116.2, + "cgroup_peak_mib": 158.2, + "cgroup_peak_sampled_mib": 146.7 }, { - "replica": "6128eb268c44", + "replica": "d590adc21943", "uploads_served": 10, "rss_before_mib": 95.0, - "rss_peak_mib": 116.1, - "rss_growth_mib": 21.0, - "rss_peak_sampled_mib": 116.1, - "cgroup_peak_mib": 155.6, - "cgroup_peak_sampled_mib": 155.6 + "rss_peak_mib": 118.6, + "rss_growth_mib": 23.6, + "rss_peak_sampled_mib": 118.6, + "cgroup_peak_mib": 167.8, + "cgroup_peak_sampled_mib": 167.7 } ] } diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt index cdcb73fd..8a71056a 100644 --- a/evidence/upload-memory.txt +++ b/evidence/upload-memory.txt @@ -1,13 +1,13 @@ # upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas -# date: 2026-10-21 21:08:48, commit: 2dbed80 + uncommitted changes +# date: 2026-10-22 14:22:25, commit: c942c61 + uncommitted changes # command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json -# exit code: 0, 34.2 s +# exit code: 0, 37.0 s client: Python 3.11.1 on Windows 10 preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)... large: 9953047 bytes; 40 Mpx: 116714 bytes -{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_201": true, "wall_seconds": 1.52, "latency_s": {"p50": 1.38, "max": 1.52}, "sampler_rounds": 9, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.5, "rss_peak_mib": 106.2, "rss_growth_mib": 12.6, "rss_peak_sampled_mib": 106.2, "cgroup_peak_mib": 134.1, "cgroup_peak_sampled_mib": 127.7}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 94.8, "rss_peak_mib": 105.3, "rss_growth_mib": 10.5, "rss_peak_sampled_mib": 105.3, "cgroup_peak_mib": 135.2, "cgroup_peak_sampled_mib": 131.4}]} -{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_201": true, "wall_seconds": 1.09, "latency_s": {"p50": 0.85, "max": 1.09}, "sampler_rounds": 8, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.2, "rss_peak_mib": 96.6, "rss_growth_mib": 3.4, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 84.3, "cgroup_peak_sampled_mib": 77.3}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 93.1, "rss_peak_mib": 94.7, "rss_growth_mib": 1.6, "rss_peak_sampled_mib": 94.6, "cgroup_peak_mib": 86.8, "cgroup_peak_sampled_mib": 77.3}]} -{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_201": true, "wall_seconds": 1.77, "latency_s": {"p50": 1.37, "max": 1.77}, "sampler_rounds": 10, "replicas": [{"replica": "11a93acecaca", "uploads_served": 10, "rss_before_mib": 94.8, "rss_peak_mib": 121.7, "rss_growth_mib": 26.9, "rss_peak_sampled_mib": 121.7, "cgroup_peak_mib": 181.9, "cgroup_peak_sampled_mib": 180.6}, {"replica": "6128eb268c44", "uploads_served": 10, "rss_before_mib": 95.0, "rss_peak_mib": 116.1, "rss_growth_mib": 21.0, "rss_peak_sampled_mib": 116.1, "cgroup_peak_mib": 155.6, "cgroup_peak_sampled_mib": 155.6}]} +{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_201": true, "wall_seconds": 3.52, "latency_s": {"p50": 2.87, "max": 3.52}, "sampler_rounds": 18, "replicas": [{"replica": "343119df2488", "uploads_served": 5, "rss_before_mib": 94.8, "rss_peak_mib": 107.6, "rss_growth_mib": 12.8, "rss_peak_sampled_mib": 107.6, "cgroup_peak_mib": 136.0, "cgroup_peak_sampled_mib": 134.5}, {"replica": "d590adc21943", "uploads_served": 5, "rss_before_mib": 91.2, "rss_peak_mib": 104.5, "rss_growth_mib": 13.3, "rss_peak_sampled_mib": 104.5, "cgroup_peak_mib": 136.2, "cgroup_peak_sampled_mib": 133.3}]} +{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_201": true, "wall_seconds": 1.09, "latency_s": {"p50": 0.85, "max": 1.09}, "sampler_rounds": 12, "replicas": [{"replica": "343119df2488", "uploads_served": 5, "rss_before_mib": 95.1, "rss_peak_mib": 96.6, "rss_growth_mib": 1.5, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 86.5, "cgroup_peak_sampled_mib": 77.6}, {"replica": "d590adc21943", "uploads_served": 5, "rss_before_mib": 92.1, "rss_peak_mib": 96.6, "rss_growth_mib": 4.6, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 83.5, "cgroup_peak_sampled_mib": 77.8}]} +{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_201": true, "wall_seconds": 1.78, "latency_s": {"p50": 1.5, "max": 1.78}, "sampler_rounds": 12, "replicas": [{"replica": "343119df2488", "uploads_served": 10, "rss_before_mib": 93.2, "rss_peak_mib": 116.2, "rss_growth_mib": 23.0, "rss_peak_sampled_mib": 116.2, "cgroup_peak_mib": 158.2, "cgroup_peak_sampled_mib": 146.7}, {"replica": "d590adc21943", "uploads_served": 10, "rss_before_mib": 95.0, "rss_peak_mib": 118.6, "rss_growth_mib": 23.6, "rss_peak_sampled_mib": 118.6, "cgroup_peak_mib": 167.8, "cgroup_peak_sampled_mib": 167.7}]} one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB -PASS: all uploads succeeded; largest RSS growth of one replica: 26.9 MiB (limit 512 MiB); results in evidence/upload-memory.json +PASS: all uploads succeeded; largest RSS growth of one replica: 23.6 MiB (limit 512 MiB); results in evidence/upload-memory.json diff --git a/student-tests/images.py b/student-tests/images.py index a83b5de8..e49bfc6f 100644 --- a/student-tests/images.py +++ b/student-tests/images.py @@ -22,7 +22,7 @@ def chunk(kind: bytes, data: bytes) -> bytes: def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes: """PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой). - pad_to — дополнить файл служебным чанком prVt до точного размера в байтах. + pad_to — дополнить файл текстовым чанком tEXt после IDAT до точного размера в байтах. idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое). """ channels = 1 if gray else 3 @@ -43,9 +43,9 @@ def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, ida idat = b"".join(parts) body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat) if pad_to is not None: - need = pad_to - len(body) - 12 - 12 # чанк-заполнитель + IEND + need = pad_to - len(body) - 12 - 12 - 8 # заголовок чанка-заполнителя, IEND, ключ "Comment\0" assert need >= 0, "image is already larger than pad_to" - body += chunk(b"prVt", bytes(need)) + body += chunk(b"tEXt", b"Comment\x00" + b"a" * need) return body + chunk(b"IEND", b"") @@ -72,3 +72,22 @@ def flip_byte(data: bytes, position: int) -> bytes: GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + + +# --- сборка PNG из произвольных чанков (для проверок смысла чанков при верных CRC) --- + +def ihdr(width, height, depth=8, color=2, interlace=0): + return b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace) + + +def zero_idat(width, height, depth=8, channels=3): + """Сжатые пиксели: каждая строка = байт фильтра 0 + нули (индекс палитры 0 / чёрный).""" + row_bytes = (width * depth * channels + 7) // 8 + return b"IDAT", zlib.compress((b"\x00" + bytes(row_bytes)) * height) + + +def assemble(*chunks): + """PNG из чанков (kind, data) с ВЕРНЫМИ CRC; IEND добавляется, если не передан.""" + if not chunks or chunks[-1][0] != b"IEND": + chunks = chunks + ((b"IEND", b""),) + return b"\x89PNG\r\n\x1a\n" + b"".join(chunk(kind, data) for kind, data in chunks) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index e1178c12..4f88f7d3 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -6,6 +6,7 @@ import hashlib import http.client import socket +import struct import urllib.parse import uuid import zlib @@ -135,6 +136,73 @@ def test_damaged_images(): print("PASS: damaged and foreign files are rejected with 415, never 500") +def test_png_chunk_semantics(): + """CRC у всех чанков верные, но содержимое нарушает спецификацию PNG -> 415, а не 201. + + Регрессия: PNG 1x1, глубина 8, палитровый, PLTE из 771 байта (257 записей) раньше получал 201 ready, + хотя Pillow его отклоняет (invalid palette size). + """ + alice, bob, _, chat = setup() + from images import assemble, ihdr, zero_idat + pal = lambda entries: (b"PLTE", bytes(3 * entries)) + idat_p = zero_idat(1, 1, 8, 1) + bad = [ + ("PLTE 771 bytes = 257 entries (review case)", + assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(771)), (b"IDAT", zlib.compress(b"\0\0")))), + ("PLTE length not a multiple of 3", assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(4)), idat_p)), + ("empty PLTE", assemble(ihdr(1, 1, 8, 3), (b"PLTE", b""), idat_p)), + ("palette larger than bit depth allows (depth 1, 3 entries)", + assemble(ihdr(1, 1, 1, 3), pal(3), zero_idat(1, 1, 1, 1))), + ("palette image without PLTE", assemble(ihdr(1, 1, 8, 3), idat_p)), + ("PLTE in a grayscale image", assemble(ihdr(1, 1, 8, 0), pal(2), zero_idat(1, 1, 8, 1))), + ("PLTE after IDAT", assemble(ihdr(1, 1, 8, 3), idat_p, pal(2))), + ("two PLTE chunks", assemble(ihdr(1, 1, 8, 3), pal(2), pal(2), idat_p)), + ("tRNS longer than the palette", assemble(ihdr(1, 1, 8, 3), pal(2), (b"tRNS", bytes(3)), idat_p)), + ("tRNS before PLTE", assemble(ihdr(1, 1, 8, 3), (b"tRNS", bytes(1)), pal(2), idat_p)), + ("tRNS of wrong size for RGB (4 bytes instead of 6)", + assemble(ihdr(1, 1, 8, 2), (b"tRNS", bytes(4)), zero_idat(1, 1, 8, 3))), + ("tRNS in an image with alpha channel", + assemble(ihdr(1, 1, 8, 6), (b"tRNS", bytes(2)), zero_idat(1, 1, 8, 4))), + ("IEND with data", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"IEND", b"x"))), + ("unknown critical chunk", assemble(ihdr(1, 1, 8, 2), (b"ABCD", b""), zero_idat(1, 1, 8, 3))), + ("IDAT chunks not consecutive", + assemble(ihdr(1, 2, 8, 2), (b"IDAT", zero_idat(1, 2, 8, 3)[1][:5]), (b"tEXt", b"k\0v"), + (b"IDAT", zero_idat(1, 2, 8, 3)[1][5:]))), + ("gAMA after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"gAMA", bytes(4)))), + ("gAMA of 2 bytes instead of 4", assemble(ihdr(1, 1, 8, 2), (b"gAMA", bytes(2)), zero_idat(1, 1, 8, 3))), + ("3 MiB metadata chunk before IDAT", assemble(ihdr(1, 1, 8, 2), (b"prVt", bytes(3 * 1024 * 1024)), + zero_idat(1, 1, 8, 3))), + ("private chunk after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"prVt", bytes(100)))), + ("animated PNG frame (fdAT)", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"fdAT", bytes(10)))), + ("tIME of 2 bytes instead of 7", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tIME", bytes(2)))), + ("tEXt without keyword separator", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tEXt", b"nokey"))), + ("zTXt with broken zlib after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0garbage"))), + ("zTXt text bomb: 50 MB of text in ~50 KB after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0" + zlib.compress(bytes(50_000_000))))), + ] + for name, data in bad: + check_error(upload(alice, chat["id"], data, filename="bad.png", expected=415), "unsupported_media_type") + print(f"{name} ({len(data)} bytes, valid CRCs) -> 415") + good = [ + ("palette 256 entries + tRNS + tEXt and tIME after IDAT", + assemble(ihdr(2, 2, 8, 3), pal(256), (b"tRNS", bytes(256)), zero_idat(2, 2, 8, 1), + (b"tEXt", b"Comment\0ok"), (b"tIME", struct.pack(">HBBBBB", 2026, 10, 22, 12, 0, 0)))), + ("palette depth 2 with 4 entries", assemble(ihdr(3, 1, 2, 3), pal(4), zero_idat(3, 1, 2, 1))), + ("RGB with 6-byte tRNS and gAMA", assemble(ihdr(1, 1, 8, 2), (b"gAMA", struct.pack(">I", 45455)), + (b"tRNS", bytes(6)), zero_idat(1, 1, 8, 3))), + ("compressed zTXt (100 KB of text) and iTXt after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"Comment\0\0" + zlib.compress(b"a" * 100_000)), + (b"iTXt", b"Title\0\1\0ru\0\0" + zlib.compress("Привет".encode())))), + ] + for name, data in good: + att = upload(alice, chat["id"], data, filename="good.png").json + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "valid PNG changed") + print(f"{name} -> 201, downloaded byte for byte") + print("PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted") + + def test_mime_substitution(): alice, _, _, chat = setup() cases = [ @@ -264,5 +332,5 @@ def test_aborted_upload(): if __name__ == "__main__": run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming, - test_pixel_limit, test_damaged_images, test_mime_substitution, test_filenames, test_access_control, + test_pixel_limit, test_damaged_images, test_png_chunk_semantics, test_mime_substitution, test_filenames, test_access_control, test_message_attachments, test_idempotency_with_attachments, test_aborted_upload]) diff --git a/uploads.py b/uploads.py index 24522125..c01bfc15 100644 --- a/uploads.py +++ b/uploads.py @@ -183,7 +183,12 @@ def inspect(upload: Upload, declared: str) -> ImageInfo: upload.file.seek(0) try: with _decode_slots: - width, height = check_png(upload.file) if detected == PNG else check_jpeg(upload.file) + if detected == PNG: + width, height = check_png(upload.file) + upload.file.seek(0) + check_png_metadata(upload.file) + else: + width, height = check_jpeg(upload.file) except TooManyPixels: raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None except Damaged as exc: @@ -262,13 +267,40 @@ class PngRows: return self.total == self.expected +# Чанки, содержимое которых проверяется (держим в памяти; размер ограничен спецификацией): +# IHDR — 13 байт, PLTE — до 256 x 3 = 768, tRNS — до 256 (палитра) / 6 (RGB). +SMALL_CHUNK_LIMIT = {b"IHDR": 13, b"PLTE": 768, b"tRNS": 256} +CRITICAL = (b"IHDR", b"PLTE", b"IDAT", b"IEND") +# После данных изображения допустимы только время и текст (строгий список). +AFTER_IDAT = {b"IEND", b"tIME", b"tEXt", b"zTXt", b"iTXt"} +TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt") +# Анимированные PNG (APNG) не поддерживаются: кадры — отдельная сущность, превью строится по одному кадру. +APNG_CHUNKS = (b"acTL", b"fcTL", b"fdAT") +# Как PngImagePlugin.MAX_TEXT_CHUNK в Pillow: больше 1 MiB распакованного текста в одном чанке — отказ. +TEXT_LIMIT = 1024 * 1024 +# Чанки метаданных до IDAT Pillow (Image.open) читает целиком: ограничиваем, чтобы память не росла. +# ICC-профили и EXIF реальных картинок — единицы-сотни KiB. +PRE_IDAT_CHUNK_LIMIT = 2 * 1024 * 1024 +# Чанки, которые по спецификации обязаны стоять ДО первого IDAT (PNG 3rd ed., 5.6 «Chunk ordering»). +# После IDAT допустимы только tIME, tEXt, zTXt, iTXt, кадры APNG и частные чанки. +BEFORE_IDAT = {b"PLTE", b"tRNS", b"cHRM", b"cICP", b"gAMA", b"iCCP", b"mDCV", b"cLLI", b"sBIT", b"sRGB", + b"bKGD", b"hIST", b"pHYs", b"sPLT", b"eXIf"} + + def check_png(file: BinaryIO) -> tuple[int, int]: + """Структура PNG по спецификации (PNG 3rd ed., гл. 5 и 11) без декодирования пикселей в память. + + Кроме CRC каждого чанка проверяется СМЫСЛ критичных чанков и tRNS: CRC защищает только от + случайной порчи, а намеренно собранный файл с верными CRC может быть некорректным. + """ if file.read(8) != PNG_SIGNATURE: raise Damaged("signature") rows = inflater = None - width = height = 0 - seen_idat = idat_finished = have_plte = False + width = height = depth = 0 color = None + palette_entries = 0 + seen = set() # какие чанки уже встречались + idat_started = idat_finished = False first = True while True: header = file.read(8) @@ -277,11 +309,37 @@ def check_png(file: BinaryIO) -> tuple[int, int]: length, kind = struct.unpack(">I4s", header) if length > 0x7FFFFFFF: raise Damaged("chunk length") + if not kind.isalpha() or not kind.isascii(): + raise Damaged("chunk type") if first and kind != b"IHDR": raise Damaged("IHDR must be first") + if kind[:1].isupper() and kind not in CRITICAL: + raise Damaged(f"unknown critical chunk {kind!r}") # такой чанк декодер обязан отвергнуть + if kind in SMALL_CHUNK_LIMIT and length > SMALL_CHUNK_LIMIT[kind]: + raise Damaged(f"{kind!r} is too long") + if kind == b"IDAT": + if rows is None or idat_finished: + raise Damaged("IDAT chunks must be consecutive") + idat_started = True + elif idat_started: + idat_finished = True + if kind in APNG_CHUNKS: + raise Damaged("animated PNG is not supported") + if kind in BEFORE_IDAT and idat_started: + raise Damaged(f"{kind!r} after IDAT") + if idat_finished and kind not in AFTER_IDAT: + raise Damaged(f"{kind!r} after IDAT") + if not idat_started and kind != b"IDAT" and length > PRE_IDAT_CHUNK_LIMIT: + raise Damaged(f"{kind!r} metadata chunk is too large") + if kind == b"tIME" and length != 7: + raise Damaged("tIME size") + if kind in (b"IHDR", b"PLTE", b"tRNS", b"IEND") and kind in seen: + raise Damaged(f"duplicate {kind!r}") + seen.add(kind) + crc = zlib.crc32(kind) - keep = kind in (b"IHDR",) kept = b"" + text = TextChunk(kind) if kind in TEXT_CHUNKS else None left = length while left: piece = file.read(min(CRC_CHUNK, left)) @@ -289,18 +347,20 @@ def check_png(file: BinaryIO) -> tuple[int, int]: raise Damaged("truncated chunk") left -= len(piece) crc = zlib.crc32(piece, crc) - if keep: + if kind in SMALL_CHUNK_LIMIT: kept += piece elif kind == b"IDAT": - if rows is None or idat_finished: - raise Damaged("IDAT out of order") feed_inflate(inflater, rows, piece) + elif text is not None: + text.feed(piece) + if text is not None: + text.finish() stored = file.read(4) if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: raise Damaged(f"CRC mismatch in {kind!r}") if kind == b"IHDR": - if not first or length != 13: + if length != 13: raise Damaged("IHDR") width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) if width > 0x7FFFFFFF or height > 0x7FFFFFFF: @@ -313,20 +373,34 @@ def check_png(file: BinaryIO) -> tuple[int, int]: rows = PngRows(width, height, depth, color, interlace) inflater = zlib.decompressobj() elif kind == b"PLTE": - have_plte = True - elif kind == b"IDAT": - seen_idat = True + # 1-256 записей по 3 байта; для палитровых — не больше 2^глубина; у серых палитры быть не может. + if color in (0, 4): + raise Damaged("PLTE in a grayscale image") + if length == 0 or length % 3: + raise Damaged("palette size") + palette_entries = length // 3 + if color == 3 and palette_entries > 1 << depth: + raise Damaged("palette larger than the bit depth allows") + elif kind == b"tRNS": + # Прозрачность: серое — 2 байта, RGB — 6, палитра — не больше записей PLTE; при альфа-канале запрещён. + if color in (4, 6): + raise Damaged("tRNS with an alpha channel") + if color == 3 and not palette_entries: + raise Damaged("tRNS before PLTE") + expected_ok = {0: length == 2, 2: length == 6, 3: 1 <= length <= palette_entries}[color] + if not expected_ok: + raise Damaged("tRNS size") elif kind == b"IEND": + if length != 0: + raise Damaged("IEND with data") break - if seen_idat and kind != b"IDAT": - idat_finished = True first = False if file.read(1): raise Damaged("data after IEND") - if color == 3 and not have_plte: + if color == 3 and not palette_entries: raise Damaged("palette image without PLTE") - if not seen_idat: + if not idat_started: raise Damaged("no IDAT") tail = inflater.flush(INFLATE_CHUNK) if tail: @@ -336,6 +410,100 @@ def check_png(file: BinaryIO) -> tuple[int, int]: return width, height +def check_png_metadata(file: BinaryIO) -> None: + """Второе мнение: стандартный парсер Pillow разбирает метаданные до IDAT (gAMA, iCCP, pHYs, tEXt…), + НЕ читая пиксели. Только open, без verify(): verify читает каждый чанк, включая IDAT до 10 MiB, + целиком в память, а CRC и данные после IDAT уже потоково проверил check_png. + Нужен для вспомогательных чанков: по спецификации их можно игнорировать, но если Pillow + не может их разобрать, файл нельзя будет обработать (превью в lab5) — лучше отклонить сразу. + """ + with Image.open(file, formats=["PNG"]): + pass # open разбирает все чанки до первого IDAT; после IDAT check_png пускает только время и текст + + +class TextChunk: + """Потоковая проверка tEXt / zTXt / iTXt: ключ 1-79 байт, метод сжатия 0, сжатый текст — + корректный zlib не больше TEXT_LIMIT после распаковки (распаковка кусками, текст не хранится).""" + + def __init__(self, kind: bytes): + self.kind = kind + self.head = b"" + self.done_header = False + self.inflater = None + self.size = 0 + + def feed(self, piece: bytes) -> None: + if not self.done_header: + self.head += piece + rest = self._header() + if rest is None: + if len(self.head) > 4096: + raise Damaged("text chunk header") + return + piece = rest + if self.inflater is not None and piece: + self._inflate(piece) + + def _keyword_end(self) -> int | None: + end = self.head.find(b"\x00") + if end == -1: + return None + if not 1 <= end <= 79: + raise Damaged("text keyword") + return end + + def _header(self) -> bytes | None: + end = self._keyword_end() + if end is None: + return None + h = self.head + if self.kind == b"tEXt": + self.done_header = True + return b"" + if self.kind == b"zTXt": + if len(h) < end + 2: + return None + if h[end + 1] != 0: + raise Damaged("zTXt compression method") + self.done_header, self.inflater = True, zlib.decompressobj() + return h[end + 2:] + # iTXt: keyword \0 flag method language \0 translated-keyword \0 text + if len(h) < end + 3: + return None + flag, method = h[end + 1], h[end + 2] + if flag not in (0, 1) or (flag and method != 0): + raise Damaged("iTXt compression") + lang_end = h.find(b"\x00", end + 3) + tkey_end = h.find(b"\x00", lang_end + 1) if lang_end != -1 else -1 + if tkey_end == -1: + return None + self.done_header = True + if flag: + self.inflater = zlib.decompressobj() + return h[tkey_end + 1:] + + def _inflate(self, data: bytes) -> None: + if self.inflater.eof: + raise Damaged("data after compressed text") + out = self.inflater.decompress(data, INFLATE_CHUNK) + self.size += len(out) + while self.inflater.unconsumed_tail and not self.inflater.eof and self.size <= TEXT_LIMIT: + out = self.inflater.decompress(self.inflater.unconsumed_tail, INFLATE_CHUNK) + self.size += len(out) + if self.size > TEXT_LIMIT: + raise Damaged("compressed text is too large") + if self.inflater.eof and self.inflater.unused_data: + raise Damaged("data after compressed text") + + def finish(self) -> None: + if not self.done_header: + raise Damaged("text chunk header") + if self.inflater is not None: + self.size += len(self.inflater.flush(INFLATE_CHUNK)) + if not self.inflater.eof or self.size > TEXT_LIMIT: + raise Damaged("compressed text") + + def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" if inflater.eof: -- 2.54.0