Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4d4b914676 | ||
|
|
5c3a31db2e |
@@ -18,3 +18,10 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
tests
|
||||
student-tests
|
||||
contracts
|
||||
scripts
|
||||
infra
|
||||
examples
|
||||
*.md
|
||||
|
||||
+25
-2
@@ -7,14 +7,37 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
|
||||
# Pools and timeouts (same as lab3)
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_WAIT_SECONDS=3
|
||||
DB_CONNECT_TIMEOUT_SECONDS=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
REDIS_POOL_MAX=20
|
||||
REDIS_TIMEOUT_MS=500
|
||||
SHUTDOWN_DRAIN_SECONDS=3
|
||||
SHUTDOWN_TIMEOUT_SECONDS=10
|
||||
|
||||
# lab4: S3 client and uploads
|
||||
S3_CONNECT_TIMEOUT_SECONDS=1
|
||||
S3_READ_TIMEOUT_SECONDS=5
|
||||
S3_MAX_ATTEMPTS=2
|
||||
S3_POOL_MAX=20
|
||||
UPLOAD_TIMEOUT_SECONDS=60
|
||||
UPLOAD_IDLE_TIMEOUT_SECONDS=10
|
||||
DECODE_CONCURRENCY=2
|
||||
# Cleanup of abandoned uploads (janitor service): grace period and pass interval
|
||||
ATTACHMENT_GRACE_SECONDS=3600
|
||||
JANITOR_INTERVAL_SECONDS=300
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
*.sh text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.example text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
@@ -1,14 +1,14 @@
|
||||
## Лабораторная 4
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Бабиченко Максим, БВТ2303
|
||||
Целевая оценка (3/4/5): 4
|
||||
Трек 5, если заявлен: не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: перенесён весь lab3 (API на FastAPI, PostgreSQL, сессии в Redis, HAProxy + 2 реплики, идемпотентность, migrate, свои тесты). Добавлены: SeaweedFS (`s3`) с приватным bucket и volume `s3data` (MinIO недоступен в публичных реестрах); идемпотентный bootstrap `s3-init`; отдельный ключ API с правами только на свой bucket; миграция 004 (`attachments`, `message_attachments` с составными FK по `chat_id`); синхронная загрузка PNG/JPEG с потоковыми лимитами (10 MiB, 40 Мпикс до распаковки, MIME по содержимому, проверка целостности без декодирования в память), скачивание через API с проверкой участника; `attachment_ids` в сообщениях; запись «строка-намерение -> PUT -> ready» и сервис `janitor` (очистка брошенных загрузок и объектов-сирот с grace period, dry-run, отчёт о missing). Подробности — REPORT.md.
|
||||
Команды запуска из чистого clone: `Copy-Item .env.example .env; python scripts/init_secrets.py; docker compose up -d --build --wait; python student-tests/collect_evidence.py` (полностью — REPORT.md, раздел «Запуск из чистого clone»).
|
||||
Результаты проверок и ссылка на REPORT.md: evidence/summary.md и evidence/*.txt; таблицы замеров — REPORT.md, раздел «Эксперименты и отказы».
|
||||
Ограничения / известные ошибки: один узел (lb, postgres, redis, s3 — SPOF); повтор загрузки после потерянного ответа создаёт второе вложение (так по контракту); missing-объекты janitor только находит. См. REPORT.md, «Ограничения».
|
||||
Источники и использование ИИ: библиотеки и документация перечислены в REPORT.md; при написании кода и тестов использовался ИИ-ассистент (Claude), код проверен и прогнан мной.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
|
||||
@@ -15,6 +15,7 @@ data/
|
||||
backups/
|
||||
# Runtime/build artifacts
|
||||
.venv/
|
||||
venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
node_modules/
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
FROM python:3.11.13-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt
|
||||
|
||||
COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py ./
|
||||
COPY migrations/ ./migrations/
|
||||
|
||||
RUN useradd --create-home --shell /usr/sbin/nologin appuser
|
||||
|
||||
USER appuser
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)"
|
||||
|
||||
CMD ["python", "main.py"]
|
||||
@@ -1,37 +1,236 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Бабиченко Максим, БВТ2303
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран
|
||||
- Commit/PR: <<ссылка на MR в Gitea после публикации>>
|
||||
- Использованные источники, библиотеки и помощь ИИ: всё из lab3 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py; образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine, haproxy:3.2.25-alpine). Новое: boto3/botocore 1.43.116 (S3-клиент), Pillow 12.3.0 (проверка JPEG), образ chrislusf/seaweedfs:3.80. Документация: AWS S3 (PutObject, x-amz-checksum-sha256, presigned URLs), SeaweedFS (`weed server -s3`, S3 API, статический конфиг identities с правами `Read/Write/List:<bucket>`), спецификация PNG (структура чанков, CRC, IHDR, фильтры строк, Adam7), botocore Config (timeouts, `total_max_attempts`). При написании кода и тестов использовался ИИ-ассистент (Claude); весь код проверен и прогнан мной, объяснить и изменить его могу на защите.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Проверял на Windows 11, все команды в PowerShell. Версии Docker, Compose и ресурсы машины — в `evidence/stack.txt`. Python нужен только для тестов (стандартная библиотека, Pillow на хосте не нужен). Перед запуском остановить стенд lab3 (`docker compose down` в его папке), иначе занят порт 8080.
|
||||
|
||||
```powershell
|
||||
git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab4.git
|
||||
cd mtusi2026lab4
|
||||
git checkout solution/bvt2303-babichenko
|
||||
$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают
|
||||
Copy-Item .env.example .env
|
||||
python scripts/init_secrets.py # случайные пароли, S3-ключи и secrets/s3_config.json (папка в .gitignore)
|
||||
docker compose up -d --build --wait # postgres, s3 -> migrate, s3-init -> 2 реплики api, janitor -> lb
|
||||
docker compose ps -a # migrate и s3-init: Exited (0), остальные healthy/running
|
||||
|
||||
python student-tests/collect_evidence.py # все проверки по очереди (~8 мин), вывод в evidence/, таблицы в этот отчёт
|
||||
```
|
||||
|
||||
Отдельные проверки (make на моём Windows нет, ниже то же самое без make):
|
||||
|
||||
| Команда из README | То же в PowerShell |
|
||||
| --- | --- |
|
||||
| `make check` | `python scripts/check_contract.py` |
|
||||
| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` |
|
||||
| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` |
|
||||
| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` |
|
||||
| тесты вложений (уровень 3–4) | `python student-tests/test_attachments.py` |
|
||||
| S3 вне API, права ключа, bootstrap | `python student-tests/test_s3_security.py` |
|
||||
| отказы S3/БД, janitor | `python student-tests/test_consistency.py [s3\|crash\|orphan\|aborted]` |
|
||||
| 10+ одновременных загрузок, память | `python student-tests/upload_memory.py` |
|
||||
| ручной проход очистки | `docker compose exec janitor python janitor.py --once --dry-run` |
|
||||
|
||||
После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть её: `docker start <ID из docker compose ps -a>` или `docker compose up -d --wait`. На Linux/macOS вместо `.cmd` — `scripts/restart.sh` и `scripts/stop-one.sh`.
|
||||
|
||||
Bootstrap: миграции применяет одноразовый сервис `migrate`, bucket и проверку прав S3 — одноразовый сервис `s3-init` (`s3_init.py`, тот же образ, что у API); оба запускаются сами при `docker compose up`, оба идемпотентны. Повторить вручную: `docker compose run --rm migrate`, `docker compose run --rm s3-init`. Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py` (новые пароли не подходят к старым volumes, поэтому удаляются вместе).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент --HTTP--> lb: HAProxy 3.2, 127.0.0.1:8080 (единственный опубликованный порт)
|
||||
| round robin, без привязки клиента к реплике
|
||||
+--> api-1 --+
|
||||
+--> api-2 --+--> postgres: пользователи, чаты, сообщения, МЕТАДАННЫЕ вложений (volume pgdata)
|
||||
+--> redis: сессии (AOF, volume redisdata)
|
||||
+--> s3 (SeaweedFS): БАЙТЫ вложений, приватный bucket (volume s3data)
|
||||
janitor: раз в 5 мин удаляет брошенные загрузки и объекты-сироты (postgres + s3)
|
||||
migrate: одноразово применяет migrations/*.sql
|
||||
s3-init: одноразово создаёт приватный bucket и проверяет права ключей и закрытость bucket
|
||||
сети: edge (lb <-> api) и data (всё остальное; internal, без выхода наружу)
|
||||
```
|
||||
|
||||
| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` на 8404 | нет | edge; `127.0.0.1:8080`, `127.0.0.1:8404/stats` | не нужны |
|
||||
| API ×2 | `api` (Dockerfile, `deploy.replicas: 2`) | python:3.11.13-slim | `/health/ready` (PostgreSQL + Redis) | нет | edge + data; наружу не публикуется | `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` |
|
||||
| S3 | `s3` | chrislusf/seaweedfs:3.80 (`weed server -s3`: master, volume, filer и S3 в одном процессе) | S3-порт отвечает по HTTP (`wget`) | `s3data` (данные и метаданные filer) | только data; S3 8333, UI master 9333 и filer 8888 доступны только внутри стенда, наружу не публикуются | `s3_config` (конфиг ключей и прав) |
|
||||
| Bootstrap S3 | `s3-init` (образ приложения, `s3_init.py`) | python:3.11.13-slim | завершается с кодом 0 | нет | data | admin-ключ + ключ API (для проверки прав) |
|
||||
| Очистка | `janitor` (тот же образ) | python:3.11.13-slim | процесс работает | нет | data | `database_url` + ограниченный ключ S3 |
|
||||
| Миграции | `migrate` | python:3.11.13-slim | завершается с кодом 0 | нет | data | только `database_url` |
|
||||
| База данных | `postgres` | postgres:17.11-alpine | `pg_isready` | `pgdata` | data, 5432 | `POSTGRES_PASSWORD_FILE` |
|
||||
| Сессии | `redis` | redis:7.4.11-alpine | `redis-cli ping` с паролем | `redisdata` (AOF) | data, 6379 | `include /run/secrets/redis_auth` |
|
||||
|
||||
Почему SeaweedFS: из трёх разрешённых вариантов MinIO отпал — его образы больше не раздаются ни с Docker Hub, ни с quay.io (проверено при сборке стенда: `pull access denied` / `unauthorized`), а закреплённый тег без доступного образа не воспроизводится. У SeaweedFS образ на Docker Hub, а права ключей задаются статическим конфигом (`-s3.config`): отдельный ключ API с правами только на свой bucket получается без отдельной утилиты администрирования. Закреплена версия 3.80, `latest` не используется.
|
||||
|
||||
Что перенесено из lab3 без изменений по смыслу: аутентификация и сессии в Redis, пользователи/чаты/сообщения, идемпотентность, HAProxy с проверкой готовности и аварийным режимом, двухшаговая остановка по SIGTERM, сервис `migrate`, свои тесты lab1–3.
|
||||
|
||||
Что изменено в lab4:
|
||||
- Новые файлы: `storage.py` (S3-клиент), `uploads.py` (приём и проверка файла), `attachments.py` (запись в две системы), `janitor.py` (очистка/сверка), `s3_init.py` (bootstrap S3), `migrations/004_attachments.sql`.
|
||||
- `main.py`: endpoint загрузки, метаданных и скачивания; `attachment_ids` в сообщениях; `db.py`: вложения и связь сообщение–вложение.
|
||||
- `compose.yaml`: сервисы `s3`, `s3-init`, `janitor`, volume `s3data`, пять новых секретов (admin-ключ, ключ API, конфиг ключей SeaweedFS); `infra/haproxy.cfg`: `timeout server 30s` (см. ниже); restart-скрипты перезапускают ещё `s3` и `janitor`.
|
||||
- `scripts/init_secrets.py` дописывает недостающие секреты (в том числе `s3_config.json` с правами ключей) и не трогает существующие.
|
||||
|
||||
### Модель данных
|
||||
|
||||
`attachments` (миграция 004): `id`, `chat_id`, `owner_id`, `filename` (только для отображения), `content_type`, `size_bytes`, `sha256`, `width`, `height`, `object_key`, `state` (`uploading` | `ready`), `created_at`, `updated_at`. `object_key` = `originals/<32 случайных hex>`: его назначает сервер, имя файла в ключ не попадает (CHECK на формат). Внешний ключ `(chat_id, owner_id) -> chat_members`: загрузить можно только в свой чат.
|
||||
|
||||
`message_attachments (message_id, chat_id, attachment_id, position)`: порядок вложений в сообщении. Оба внешних ключа составные и содержат один и тот же `chat_id`: `(message_id, chat_id) -> messages`, `(attachment_id, chat_id) -> attachments`. Поэтому вложение другого чата нельзя прикрепить даже прямым INSERT в обход API. Одно вложение можно прикрепить к нескольким сообщениям (контракт: «любой участник может ссылаться на доступное вложение чата»). Ограничение на текст сообщения ослаблено до `<= 16000 байт`: пустой текст допустим с вложением, проверку «текст или вложение» делает API.
|
||||
|
||||
### Загрузка: что и в каком порядке проверяется
|
||||
|
||||
`POST /api/v1/chats/{chat_id}/attachments?filename=...`, тело — сырые байты одного PNG/JPEG.
|
||||
|
||||
1. До чтения тела: сессия (401) -> участник чата (404) -> `filename` 1–255 символов без `/`, `\`, NUL, управляющих символов, не `.`/`..` (400) -> `Content-Type` только `image/png`/`image/jpeg` (415) -> `Content-Length` больше 10 485 760 (413). При этих ошибках тело дочитывается и выбрасывается (до 20 MiB / 10 с, ничего не сохраняя), чтобы клиент получил ответ, а не обрыв соединения.
|
||||
2. Тело читается кусками: размер и SHA-256 считаются на лету, данные пишутся во временный файл (до 256 KiB в памяти, остальное на диск контейнера). Первый байт сверх 10 MiB -> 413, чтение прекращается (работает и без `Content-Length`, при `Transfer-Encoding: chunked`). Таймауты: 10 с без данных, 60 с на всё тело (`UPLOAD_IDLE_TIMEOUT_SECONDS`, `UPLOAD_TIMEOUT_SECONDS`).
|
||||
3. Тип по сигнатуре (`\x89PNG...`, `FF D8 FF`), не по заголовку. Не PNG/JPEG -> 415; заявленный тип не совпадает с содержимым -> 415.
|
||||
4. Размер в пикселях читается из заголовка (IHDR / SOF) и проверяется ДО распаковки: больше 40 000 000 -> 413.
|
||||
5. Проверка целостности с ограниченной памятью:
|
||||
- PNG — своя потоковая проверка (`uploads.check_png`): CRC каждого чанка и смысл критичных чанков по спецификации PNG (размер палитры 1–256 записей и не больше 2^глубина, размер tRNS по типу цвета, пустой IEND, порядок чанков: всё, что обязано идти до IDAT, — до него, IDAT подряд, неизвестный критичный чанк запрещён); IDAT распаковывается кусками по 64 KiB с подсчётом точного объёма и проверкой байта фильтра каждой строки. Затем Pillow разбирает метаданные до IDAT (Image.open, без verify: verify читает каждый чанк, включая IDAT до 10 MiB, в память). После IDAT допускаются только tIME, tEXt, zTXt, iTXt; текстовые чанки проверяются потоково (ключ 1–79 байт, корректный zlib, не больше 1 MiB распакованного текста); анимированные PNG (acTL/fcTL/fdAT) отклоняются; чанк метаданных до IDAT — не больше 2 MiB. Исправлено по замечанию: раньше проверялись только CRC, и PNG с палитрой из 257 записей получал 201.
|
||||
- JPEG — Pillow декодирует весь поток, но в масштабе 1/8 (`draft`): 40 Мпикс -> ~0,6 Мпикс, около 2 MiB.
|
||||
- Одновременно в реплике проверяется не больше `DECODE_CONCURRENCY=2` файлов (CPU).
|
||||
Битый файл -> 415. Любая ошибка разбора ловится и превращается в 415, а не 500. Проверка PNG сверена с Pillow: 400 случайных корректных PNG разных типов приняты, 400 испорченных (обрезка, перевёрнутый бит) отклонены.
|
||||
6. Запись (см. следующий раздел) и ответ `201 Attachment` со `status=ready`, `variants=[]`, `error_code=null`.
|
||||
|
||||
Временный файл удаляется в `finally` при любом исходе: оригинал после ответа существует только в S3. У контейнеров `api` нет volumes, это проверяет `test_consistency.py`.
|
||||
|
||||
Скачивание: `GET /api/v1/attachments/{id}/content` — сначала проверка участника чата в PostgreSQL (иначе 404, как у несуществующего), потом GET из S3 и потоковая отдача кусками по 64 KiB с истинным `Content-Type`, `Content-Length`, `X-Content-Type-Options: nosniff`, `Content-Disposition: inline; filename*=UTF-8''...`. Без redirect и без подписанных URL. `variant` — только `original` (иное -> 400).
|
||||
|
||||
### Согласованность PostgreSQL и S3
|
||||
|
||||
Распределённой транзакции нет, поэтому порядок записи выбран так, чтобы у каждого объекта в bucket была строка-указатель в БД (`attachments.py`):
|
||||
|
||||
| Шаг | Если здесь отказ |
|
||||
| --- | --- |
|
||||
| 1. `INSERT attachments (state='uploading')` + COMMIT | в S3 ничего нет; клиенту 503 |
|
||||
| 2. `PUT` объекта в S3 с `x-amz-checksum-sha256` (S3 сам сверяет SHA-256) | откат: короткий DELETE объекта (PUT мог пройти при таймауте) и удаление строки; если S3 недоступен и откат не удался, строка `uploading` остаётся указателем для janitor; клиенту 503 |
|
||||
| 3. `UPDATE state='ready'` + COMMIT | клиенту 503; объект сразу НЕ удаляется: при обрыве связи COMMIT мог пройти, удаление сломало бы готовое вложение. Строка остаётся `uploading` -> её разберёт janitor |
|
||||
| 4. `201` | только после шага 3 |
|
||||
|
||||
Строки `uploading` не видны через API (GET -> 404) и не прикрепляются к сообщениям (404), сообщение может ссылаться только на `ready` (проверка под `FOR SHARE` в той же транзакции, что и INSERT сообщения).
|
||||
|
||||
`janitor.py` (сервис `janitor`, раз в `JANITOR_INTERVAL_SECONDS=300`, один экземпляр за счёт `pg_try_advisory_lock`):
|
||||
1. Строки `uploading` старше grace period: под `FOR UPDATE SKIP LOCKED` удалить объект (DELETE идемпотентен), затем строку. Если S3 недоступен — транзакция откатывается, строка остаётся до следующего прохода.
|
||||
2. Обход bucket (`originals/`, постранично): объект старше grace period без строки в БД (в любом состоянии) — сирота, удаляется (перед удалением ещё раз проверяется, что строка не появилась). Живая загрузка всегда имеет строку, потому что строка коммитится до PUT.
|
||||
3. Отчёт о `ready`-строках без объекта (`missing_objects`). Их janitor не удаляет: это потеря данных, её чинят восстановлением из резервной копии, а не автоматической очисткой; скачивание такого файла отвечает 503, не 500.
|
||||
|
||||
Grace period: `ATTACHMENT_GRACE_SECONDS=3600`, janitor и API отказываются стартовать, если он меньше `2 x UPLOAD_TIMEOUT_SECONDS`. Самая долгая загрузка после записи строки живёт: PUT до 2 x (1 + 5) с + откат 2 с + UPDATE до 8 с, то есть меньше 30 с — в 120 раз меньше grace period. Поэтому janitor не может удалить загрузку, которая ещё идёт, а если бы реплика зависла дольше часа, её UPDATE не найдёт строку (`state='uploading'` уже удалена под блокировкой) и клиент получит 503, а не «ready без объекта». Есть `--dry-run` (только отчёт) и `--once`.
|
||||
|
||||
Обрыв загрузки клиентом: строка пишется только после приёма и проверки всего тела, поэтому оборванная загрузка не оставляет ни строки, ни объекта; временный файл удаляется сразу.
|
||||
|
||||
### Готовность и отказ S3
|
||||
|
||||
S3 не входит в `/health/ready`: без него не работают только загрузка и скачивание файлов, а вход, чаты, сообщения (в том числе со ссылкой на уже загруженные вложения) и метаданные вложений продолжают работать. Если бы readiness зависел от S3, отказ S3 вывел бы из балансировки все реплики и остановил бы весь мессенджер. При недоступном S3 файловые операции отвечают `503 unavailable` без подробностей. Критичны для readiness, как и в lab3, PostgreSQL и Redis.
|
||||
|
||||
S3-клиент: connect timeout 1 с, read timeout 5 с, всего 2 попытки (`total_max_attempts`), откат — 1 попытка с таймаутом 2 с. Худший ответ при зависшем S3 около 13 с, поэтому в HAProxy `timeout server` увеличен с 15 до 30 с (пока клиент передаёт тело файла, этот таймаут не срабатывает: передачу ограничивают таймауты самого API).
|
||||
|
||||
### Безопасность хранилища
|
||||
|
||||
- Bucket приватный: в конфиге ключей SeaweedFS нет identity `anonymous`, поэтому любой запрос без подписи получает 403. Проверка вне API: `s3-init` и `test_s3_security.py` делают анонимные GET/HEAD/LIST/PUT прямо в S3 из внутренней сети — 403.
|
||||
- Два ключа (`secrets/s3_config.json`, генерирует `scripts/init_secrets.py`): `admin` (Admin/Read/Write/List) есть только у `s3` (в конфиге) и `s3-init`; `messenger-api` для api и janitor — только `Read:messenger-attachments`, `Write:messenger-attachments`, `List:messenger-attachments`. Создать/удалить bucket, поменять политику или versioning ключ API не может — проверяют `s3-init` и `test_s3_security.py`.
|
||||
- Порты S3 (8333), master (9333) и filer (8888) не публикуются, SeaweedFS только во внутренней сети `data` (`internal: true`); веб-интерфейсы master/filer доступны только контейнерам стенда, с хоста — connection refused.
|
||||
- Bootstrap идемпотентен: `create_bucket` при существующем bucket получает `BucketAlreadyOwnedByYou/BucketAlreadyExists` и продолжает; проверочный объект ключа API создаётся и удаляется; в конце скрипт проверяет, что bucket private. Повторный `docker compose run --rm s3-init` ничего не ломает (тест). Права ключей — декларативный конфиг, применяется при старте `s3`.
|
||||
- Приватный bucket и случайный ключ не заменяют проверку доступа в API: ключ объекта не секрет (может попасть в лог, бэкап, ошибку), а bucket «приватен» для всех, у кого нет ключа S3, — но не различает пользователей мессенджера. Кто участник какого чата, знает только API, поэтому проверка участника делается на каждый GET метаданных и байтов.
|
||||
|
||||
### Соответствие `.env.example`
|
||||
|
||||
| Переменная | Что делает в этом решении |
|
||||
| --- | --- |
|
||||
| APP_HOST, APP_PORT, APP_ENV | где слушает API (0.0.0.0:8080 внутри контейнера), APP_ENV пишется в лог |
|
||||
| SESSION_TTL_SECONDS, SESSION_STORE=redis | срок жизни сессии 3600 с, сессии в Redis (как в lab3) |
|
||||
| DATABASE_URL_FILE, CACHE_URL_FILE | пути к секретам с адресами PostgreSQL и Redis |
|
||||
| API_REPLICAS | число реплик `api`, по умолчанию 2 |
|
||||
| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) |
|
||||
| S3_ENDPOINT, S3_REGION, S3_BUCKET | `http://s3:8333`, `us-east-1`, `messenger-attachments` (path-style адресация) |
|
||||
| S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE | ограниченный ключ API (identity `messenger-api` в `s3_config.json`) |
|
||||
| MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS | 10 485 760 байт и 40 000 000 пикселей |
|
||||
| S3_CONNECT_TIMEOUT_SECONDS, S3_READ_TIMEOUT_SECONDS, S3_MAX_ATTEMPTS, S3_POOL_MAX | 1 с, 5 с, 2 попытки всего, 20 соединений |
|
||||
| UPLOAD_TIMEOUT_SECONDS, UPLOAD_IDLE_TIMEOUT_SECONDS | 60 с на всё тело, 10 с без данных |
|
||||
| DECODE_CONCURRENCY | одновременных проверок изображений в реплике, 2 |
|
||||
| ATTACHMENT_GRACE_SECONDS, JANITOR_INTERVAL_SECONDS | grace period 3600 с, проход janitor раз в 300 с |
|
||||
| DB_*, REDIS_*, SHUTDOWN_* | пулы, таймауты и остановка (как в lab3) |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Все файлы `evidence/*.txt` создаёт `python student-tests/collect_evidence.py` (вывод тестов как есть, с датой, commit и кодом выхода); сводка — `evidence/summary.md`.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS через балансировщик, с загрузкой, сообщением с `attachment_ids`, скачиванием и 404 постороннему | evidence/smoke.txt |
|
||||
| Skeleton и контракт не изменены | `python scripts/check_contract.py` | PASS; `contracts/`, `tests/`, README не менялись | evidence/check.txt |
|
||||
| 3: возможности lab3 сохранены | `test_balancing.py`, `test_idempotency.py`, `test_api.py`, `test_race.py`, `test_db.py`, `redis_info.py`, `explain.py`, `test_failures.py`, публичный failover | все PASS через балансировщик | evidence/balancing.txt, idempotency.txt, student-tests.txt, race.txt, db.txt, redis.txt, explain.txt, failures.txt, failover.txt |
|
||||
| 3: S3 в общем стенде; метаданные в PostgreSQL, байты в приватном bucket, не в БД и не в каталоге API | `check_stack.py`, `test_s3_security.py`, `test_consistency.py aborted` | `s3` и `s3-init` в compose; у `api` нет volumes, временных файлов после запросов 0; анонимный доступ к bucket 403 | evidence/stack.txt, s3-security.txt, consistency.txt |
|
||||
| 3: POST -> 201 ready после сохранения; оригинал побайтно; message с attachment_ids читается участниками; посторонний без доступа | smoke, `test_attachments.py` (roundtrip, access, message) | SHA-256 и размер совпадают, скачивает второй участник, посторонний 404 на метаданные и байты | evidence/smoke.txt, attachments.txt |
|
||||
| 3: реальные PNG/JPEG ≤10 MiB и ≤40 Мпикс; MIME по содержимому; опасные filename; лимиты до неограниченного чтения/декодирования | `test_attachments.py` | ровно 10 MiB -> 201, +1 байт -> 413 (с Content-Length и chunked); 8000x5000 -> 201, 8001x5000 -> 413, «бомба» 100000x100000 -> 413 по заголовку; 13 опасных имён -> 400 | evidence/attachments.txt |
|
||||
| 3: upload -> остановка обслужившей реплики -> download через другую | публичный `make test-failover` (lab4 загружает файл до отказа и скачивает после) для каждой реплики; `test_balancing.py` | PASS для обеих реплик; файл, загруженный через одну реплику, скачан через другую | evidence/failover.txt, balancing.txt |
|
||||
| 3: restart с сохранением volumes сохраняет метаданные и оригинал | `run_persistence.py` (публичный persistence + ID/время старта контейнеров) | postgres, redis, s3, обе api, janitor, lb перезапущены (те же контейнеры, новое время старта); файл скачивается с прежним токеном | evidence/persistence.txt |
|
||||
| 4: тесты: слишком большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; нет 500 | `test_attachments.py` | 413; 10 видов битых файлов -> 415; 7 подмен типа -> 415; неизвестный id и вложение другого чата (автор состоит в обоих) -> 404; дубли, 11 id -> 400; ни одного 500 | evidence/attachments.txt |
|
||||
| 4: прерывание загрузки и отказ между S3/БД не оставляют вечных неучтённых объектов; механизм с grace period | `test_consistency.py` (aborted, crash, orphan, s3) | обрыв: строк и объектов не прибавилось; падение после PUT: строка `uploading` невидима, janitor не трогает её до grace, `--dry-run` только отчёт, затем удалены объект и строка, повтор — 0; сирота удаляется только после grace; `ready` не трогаются; missing — только отчёт | evidence/consistency.txt |
|
||||
| 4: недоступность S3 | `test_consistency.py s3` (`docker compose stop s3` и `pause s3`) | файловые операции 503 без подробностей, остальное работает; после восстановления файлы на месте, мусор убран | evidence/consistency.txt |
|
||||
| 4: лимиты/таймауты и потоковая обработка; ≥10 одновременных загрузок с пиком памяти | `upload_memory.py` | таблица ниже | evidence/upload-memory.txt, upload-memory.json |
|
||||
| 4: bootstrap идемпотентен | `test_s3_security.py` (повторный `docker compose run --rm s3-init`) | exit 0, «bucket is private», старые файлы скачиваются, новая загрузка 201 | evidence/s3-security.txt |
|
||||
| 4: отдельные S3 credentials с ограниченными правами | `test_s3_security.py`, `check_stack.py`, `s3-init` | ключ API: get/put/delete/list своего bucket — да; create/delete bucket, policy, versioning — отказ; admin-ключ и конфиг ключей только у s3 и s3-init | evidence/s3-security.txt, stack.txt |
|
||||
| 4: консоль не открыта; проверка private bucket вне API | `test_s3_security.py`, логи `s3-init` | 127.0.0.1:8333/9333/8888 — connection refused, s3 только в сети data; анонимные GET/HEAD/LIST/PUT/policy прямо в S3 -> 403 | evidence/s3-security.txt |
|
||||
| Выбранный трек 5 | — | не выбран | — |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотезы:
|
||||
1. Память API-реплики почти не зависит от числа и размера одновременных загрузок: тело идёт во временный файл, PNG проверяется потоково, JPEG — в масштабе 1/8, проверок одновременно не больше 2 на реплику. Наивный вариант (тело в памяти + полное декодирование 40 Мпикс ~160 MiB) при 10 одновременных загрузках не поместился бы в лимит 512 MiB.
|
||||
2. Отказ S3 затрагивает только операции с байтами файлов и не оставляет видимых или вечных «половинчатых» вложений.
|
||||
|
||||
Методика памяти (`student-tests/upload_memory.py`): три сценария — 10 x PNG из случайного шума ~9,5 MiB (почти несжимаемый, максимум трафика); 10 x PNG 8000x5000 RGB (40 Мпикс, максимум распаковки); 20 смешанных. Перед каждым сценарием `docker compose restart api` (сбрасывает пик), прогрев 8 загрузками, затем все потоки стартуют одновременно (barrier) через балансировщик. Пик RSS — `VmHWM` из `/proc/1/status` процесса API (максимум за жизнь процесса по данным ядра, короткие всплески не теряются), дополнительно пик памяти cgroup контейнера и опрос раз в ~0,1–0,5 с. Проверяются коды 201 и SHA-256 скачанных файлов. Клиент — тот же компьютер, поэтому время загрузки ограничено и клиентом; это замер памяти, а не пропускной способности.
|
||||
|
||||
<!-- measurements:begin -->
|
||||
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-22, commit c942c61 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB.
|
||||
|
||||
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
|
||||
|
||||
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 1 | 343119df2488 | 2 | 123 | 0 | 4.422 |
|
||||
| 2 | d590adc21943 | 2 | 124 | 0 | 4.531 |
|
||||
|
||||
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
|
||||
|
||||
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 2.87 / 3.52 | 343119df2488: 5 | 94.8 | 107.6 | 12.8 | 136.0 |
|
||||
| | | | | d590adc21943: 5 | 91.2 | 104.5 | 13.3 | 136.2 |
|
||||
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 343119df2488: 5 | 95.1 | 96.6 | 1.5 | 86.5 |
|
||||
| | | | | d590adc21943: 5 | 92.1 | 96.6 | 4.6 | 83.5 |
|
||||
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.5 / 1.78 | 343119df2488: 10 | 93.2 | 116.2 | 23.0 | 158.2 |
|
||||
| | | | | d590adc21943: 10 | 95.0 | 118.6 | 23.6 | 167.8 |
|
||||
|
||||
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
|
||||
|
||||
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.325 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.422, 4.531 с (hook ждёт остановки ~4 с); потерь нет |
|
||||
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 47.0 с до готовности; потерь нет |
|
||||
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19453 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
|
||||
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14891 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
|
||||
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
|
||||
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
|
||||
<!-- measurements:end -->
|
||||
|
||||
Вывод по памяти: рост RSS реплики при 10–20 одновременных загрузках — единицы MiB (см. таблицу), а не «размер файла x число загрузок»; верхняя оценка на загрузку — 256 KiB буфера временного файла + 64 KiB куска чтения + буферы boto3 при PUT; на проверку — 64 KiB распаковки PNG или ~2 MiB для JPEG, не больше 2 одновременно. Пик cgroup выше пика RSS, потому что в него входит page cache временных файлов (ядро освобождает его при нехватке памяти, OOM он не вызывает).
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Выдерживается отказ одной API-реплики и временная недоступность S3 (файловые операции 503, остальное работает). Не выдерживается отказ единственных `lb`, `postgres`, `redis`, `s3` и всей машины: стенд на одном узле.
|
||||
- Повтор загрузки после потерянного ответа создаёт второе вложение (в контракте нет ключа идемпотентности для загрузки). Лишнее вложение, не прикреплённое к сообщению, остаётся: удаления в API нет; janitor удаляет только незавершённые загрузки.
|
||||
- `missing_objects` janitor только находит; восстановление — из резервной копии S3, которой в стенде нет.
|
||||
- Обход bucket janitor’ом держит в памяти список ключей (для учебного объёма — мегабайты); для миллионов объектов нужна сверка по диапазонам ключей или S3 Inventory.
|
||||
- Временный файл загрузки (часть тела сверх 256 KiB) лежит на диске контейнера. В Linux `tempfile.TemporaryFile` создаётся уже без имени в каталоге (O_TMPFILE), поэтому даже при SIGKILL реплики место освобождается ядром, «забытых» файлов не бывает. Одновременно на диске может быть до 10 MiB на каждую идущую загрузку; лимита на число одновременных загрузок, кроме лимитов HAProxy/uvicorn, нет.
|
||||
- Проверяется структура изображения, а не его безопасность для всех декодеров (например, JPEG с некорректными, но декодируемыми данными пройдёт). Перекодирование оригинала — задача lab5.
|
||||
- PNG с частными служебными чанками после IDAT и анимированные PNG отклоняются (строже спецификации, на практике не встречаются в фотографиях).
|
||||
- MinIO собирался быть первым выбором, но его образы больше не доступны в публичных реестрах: закреплённый тег защищает от неожиданного обновления, но не от исчезновения образа. Для эксплуатации нужно зеркалить образы в свой реестр. SeaweedFS запущен одним процессом `weed server` на одном диске — это SPOF и не защита от потери диска; веб-интерфейсы master/filer без авторизации, закрыты только сетью.
|
||||
- Скрипты `failover_metrics.py`, `rolling_update.py`, `test_expiry.py` из lab3 остались в `student-tests/` и работают со стендом, но в общий прогон lab4 не входят (это уровень 4 lab3, а не lab4).
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Сохранение вложения: PostgreSQL (метаданные) + S3 (байты) без распределённой транзакции.
|
||||
|
||||
Порядок записи и что происходит при отказе на каждом шаге:
|
||||
|
||||
1. INSERT attachments (state=uploading) и COMMIT — упало: в S3 ничего нет, клиенту 503.
|
||||
2. PUT объекта в S3 — упало: удаляем объект (на случай, если PUT
|
||||
на самом деле прошёл) и строку, клиенту 503.
|
||||
Если и это не вышло — остаётся uploading-строка,
|
||||
её уберёт janitor после grace period.
|
||||
3. UPDATE state=ready и COMMIT — упало/неизвестно: клиенту 503; строка остаётся
|
||||
uploading, объект есть -> janitor удалит оба.
|
||||
Объект сразу НЕ удаляем: при обрыве связи COMMIT
|
||||
мог пройти, и удаление сломало бы ready-вложение.
|
||||
4. 201 клиенту — только после шага 3.
|
||||
|
||||
Клиенту видны и прикрепляемы только ready-строки, поэтому незавершённая загрузка ни на что не влияет.
|
||||
"""
|
||||
import logging
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import db
|
||||
import storage
|
||||
from errors import unavailable
|
||||
from uploads import ImageInfo, Upload
|
||||
|
||||
log = logging.getLogger("messenger.attachments")
|
||||
|
||||
|
||||
def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict:
|
||||
attachment_id = uuid4()
|
||||
key = storage.new_key(uuid4().hex)
|
||||
db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type,
|
||||
upload.size, upload.sha256, image.width, image.height, key)
|
||||
try:
|
||||
storage.put(key, upload.file, upload.size, image.content_type, upload.sha256)
|
||||
except storage.StorageUnavailable:
|
||||
compensate(attachment_id, key)
|
||||
raise unavailable("File storage is temporarily unavailable") from None
|
||||
row = db.finish_upload(attachment_id)
|
||||
if row is None:
|
||||
# janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace).
|
||||
log.warning("Upload %s was reclaimed before it finished", attachment_id)
|
||||
raise unavailable()
|
||||
return row
|
||||
|
||||
|
||||
def compensate(attachment_id: UUID, key: str) -> None:
|
||||
"""Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor."""
|
||||
try:
|
||||
storage.delete(key, quick=True)
|
||||
db.delete_upload_intent(attachment_id)
|
||||
log.info("Upload %s rolled back after S3 failure", attachment_id)
|
||||
except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту
|
||||
log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__)
|
||||
|
||||
|
||||
def public(row: dict) -> dict:
|
||||
"""Строка БД -> Attachment из контракта (без object_key и размеров в пикселях)."""
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"owner_id": row["owner_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": "ready",
|
||||
"variants": [],
|
||||
"error_code": None,
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import logging
|
||||
import secrets
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
import settings
|
||||
from errors import BASIC_CHALLENGE, unauthorized, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.auth")
|
||||
|
||||
hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1)
|
||||
|
||||
DUMMY_HASH = hasher.hash("there-is-no-such-user")
|
||||
|
||||
PASSWORD_MIN_LENGTH = 12
|
||||
PASSWORD_MAX_LENGTH = 128
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return hasher.hash(password)
|
||||
|
||||
|
||||
def password_matches(password_hash: str | None, password: str) -> bool:
|
||||
try:
|
||||
hasher.verify(password_hash or DUMMY_HASH, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
return password_hash is not None
|
||||
|
||||
def parse_basic(header: str | None) -> tuple[str, str] | None:
|
||||
"""'Basic base64(username:password)' -> (username, password) или None.
|
||||
|
||||
Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617).
|
||||
"""
|
||||
scheme, _, value = (header or "").strip().partition(" ")
|
||||
if scheme.lower() != "basic" or not value or len(value) > 1024:
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
return None
|
||||
username, colon, password = decoded.partition(":")
|
||||
if not colon or not username or not username.isascii():
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(header: str | None) -> str | None:
|
||||
scheme, _, value = (header or "").strip().partition(" ")
|
||||
value = value.strip()
|
||||
if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
|
||||
KEY_PREFIX = "session:"
|
||||
|
||||
_redis = redis.Redis(
|
||||
connection_pool=redis.BlockingConnectionPool.from_url(
|
||||
settings.REDIS_URL or "redis://localhost:6379/0",
|
||||
max_connections=settings.REDIS_POOL_MAX,
|
||||
timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
socket_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
decode_responses=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
user_id: UUID
|
||||
key: str
|
||||
|
||||
|
||||
def session_key(token: str) -> str:
|
||||
return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def create_session(user_id: UUID) -> tuple[str, datetime]:
|
||||
"""Новая сессия: случайный токен и абсолютное время истечения."""
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS)
|
||||
try:
|
||||
_redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000))
|
||||
except redis.RedisError as exc:
|
||||
log.warning("Redis is unavailable on login: %s", type(exc).__name__)
|
||||
raise unavailable() from None
|
||||
return token, expires_at
|
||||
|
||||
|
||||
def check_bearer(header: str | None) -> Session:
|
||||
token = parse_bearer(header)
|
||||
if token is None:
|
||||
raise unauthorized("Bearer token is required")
|
||||
key = session_key(token)
|
||||
try:
|
||||
user_id = _redis.get(key)
|
||||
except redis.RedisError as exc:
|
||||
log.warning("Redis is unavailable: %s", type(exc).__name__)
|
||||
raise unavailable() from None
|
||||
if user_id is None:
|
||||
raise unauthorized("Session is invalid or expired")
|
||||
return Session(UUID(user_id), key)
|
||||
|
||||
|
||||
def delete_session(session: Session) -> None:
|
||||
try:
|
||||
_redis.delete(session.key)
|
||||
except redis.RedisError:
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def ping_redis() -> None:
|
||||
try:
|
||||
_redis.ping()
|
||||
except redis.RedisError:
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def login_failed():
|
||||
return unauthorized("Invalid username or password", BASIC_CHALLENGE)
|
||||
|
||||
|
||||
def close() -> None:
|
||||
_redis.close()
|
||||
_redis.connection_pool.disconnect()
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
name: messenger-lab4
|
||||
|
||||
x-app-build: &app-build
|
||||
context: .
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-dev}
|
||||
|
||||
x-s3-api-secrets: &s3-api-secrets
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
|
||||
services:
|
||||
migrate:
|
||||
build: *app-build
|
||||
command: ["python", "migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
|
||||
# S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений.
|
||||
# Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config
|
||||
# (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket.
|
||||
# Анонимного доступа нет: identity "anonymous" в конфиге не объявлена.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.80
|
||||
entrypoint: ["/usr/bin/weed"]
|
||||
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0",
|
||||
"-master.volumeSizeLimitMB=1024", "-volume.max=0",
|
||||
"-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"]
|
||||
secrets: [s3_config]
|
||||
volumes: ["s3data:/data"]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
# S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт»)
|
||||
test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 15s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
|
||||
# Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки.
|
||||
s3-init:
|
||||
build: *app-build
|
||||
command: ["python", "s3_init.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M}
|
||||
|
||||
api:
|
||||
build: *app-build
|
||||
env_file: .env
|
||||
environment:
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
<<: *s3-api-secrets
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [edge, data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
|
||||
# Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ.
|
||||
janitor:
|
||||
build: *app-build
|
||||
command: ["python", "janitor.py"]
|
||||
env_file: .env
|
||||
environment:
|
||||
CACHE_URL_FILE: "" # janitor не ходит в Redis
|
||||
<<: *s3-api-secrets
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 192M}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.25-alpine
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
- "127.0.0.1:8404:8404"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
networks: [edge]
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
|
||||
postgres:
|
||||
image: postgres:17.11-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"]
|
||||
secrets: [postgres_password]
|
||||
volumes: ["pgdata:/var/lib/postgresql/data"]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
|
||||
redis:
|
||||
image: redis:7.4.11-alpine
|
||||
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
|
||||
volumes:
|
||||
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
- redisdata:/data
|
||||
secrets: [redis_auth]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
data:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
redisdata: {}
|
||||
s3data: {}
|
||||
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
redis_auth: {file: ./secrets/redis_auth}
|
||||
s3_root_user: {file: ./secrets/s3_root_user}
|
||||
s3_root_password: {file: ./secrets/s3_root_password}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
s3_config: {file: ./secrets/s3_config.json}
|
||||
@@ -0,0 +1,359 @@
|
||||
import logging
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors as pg_errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
import settings
|
||||
from errors import bad_request, conflict, not_found, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.db")
|
||||
|
||||
|
||||
pool = ConnectionPool(
|
||||
conninfo=settings.DATABASE_URL,
|
||||
min_size=settings.DB_POOL_MIN,
|
||||
max_size=settings.DB_POOL_MAX,
|
||||
timeout=settings.DB_POOL_WAIT_SECONDS,
|
||||
kwargs={
|
||||
"connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||
"row_factory": dict_row,
|
||||
"options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}"
|
||||
" -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC",
|
||||
},
|
||||
check=ConnectionPool.check_connection,
|
||||
open=False,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection(wait_seconds: float | None = None):
|
||||
try:
|
||||
with pool.connection(timeout=wait_seconds) as conn:
|
||||
yield conn
|
||||
except (psycopg.OperationalError, PoolTimeout) as exc:
|
||||
log.warning("PostgreSQL is unavailable: %s", type(exc).__name__)
|
||||
raise unavailable() from None
|
||||
except psycopg.DataError as exc:
|
||||
log.warning("PostgreSQL rejected a value: %s", type(exc).__name__)
|
||||
raise bad_request("Invalid request data") from None
|
||||
|
||||
|
||||
TEXT_FIELDS = ("display_name", "title", "text", "filename")
|
||||
|
||||
|
||||
def to_db(value: str) -> bytes:
|
||||
return value.encode("utf-8")
|
||||
|
||||
|
||||
def from_db(row: dict | None) -> dict | None:
|
||||
"""Строка из БД -> dict для API: байты текстовых полей обратно в str."""
|
||||
if row is None:
|
||||
return None
|
||||
return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()}
|
||||
|
||||
|
||||
def ping() -> None:
|
||||
with connection(wait_seconds=1) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
|
||||
MIGRATIONS_LOCK = 20260202
|
||||
|
||||
|
||||
def migration_files() -> list:
|
||||
return sorted(settings.MIGRATIONS_DIR.glob("*.sql"))
|
||||
|
||||
|
||||
def run_migrations() -> list[str]:
|
||||
files = migration_files()
|
||||
applied = []
|
||||
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,))
|
||||
try:
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now())""")
|
||||
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for path in files:
|
||||
if path.name in done:
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
|
||||
applied.append(path.name)
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,))
|
||||
return applied
|
||||
|
||||
|
||||
def wait_and_migrate(max_wait_seconds: int = 60) -> None:
|
||||
deadline = time.monotonic() + max_wait_seconds
|
||||
while True:
|
||||
try:
|
||||
applied = run_migrations()
|
||||
log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)")
|
||||
return
|
||||
except psycopg.OperationalError:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log.warning("PostgreSQL is not ready yet, retry in 1 s")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def missing_migrations() -> list[str]:
|
||||
expected = [path.name for path in migration_files()]
|
||||
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
|
||||
try:
|
||||
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
except pg_errors.UndefinedTable:
|
||||
return expected
|
||||
return [name for name in expected if name not in done]
|
||||
|
||||
|
||||
def wait_for_schema(max_wait_seconds: int) -> None:
|
||||
deadline = time.monotonic() + max_wait_seconds
|
||||
while True:
|
||||
try:
|
||||
missing = missing_migrations()
|
||||
except psycopg.OperationalError:
|
||||
missing = None
|
||||
if missing == []:
|
||||
log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files()))
|
||||
return
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}")
|
||||
log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None
|
||||
else "missing " + ", ".join(missing))
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
|
||||
USER_COLUMNS = "id, username, display_name, created_at"
|
||||
|
||||
|
||||
def create_user(username: str, display_name: str, password_hash: str) -> dict:
|
||||
try:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(
|
||||
f"""INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""",
|
||||
(uuid4(), username, to_db(display_name), password_hash),
|
||||
).fetchone())
|
||||
except pg_errors.UniqueViolation:
|
||||
raise conflict("Username already exists") from None
|
||||
|
||||
|
||||
def get_user(user_id: UUID) -> dict | None:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone())
|
||||
|
||||
|
||||
def get_user_with_password(username: str) -> dict | None:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(
|
||||
f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,)
|
||||
).fetchone())
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids"""
|
||||
|
||||
|
||||
def create_chat(title: str, member_ids: list[UUID]) -> dict:
|
||||
chat_id = uuid4()
|
||||
members = sorted(member_ids)
|
||||
try:
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
|
||||
(chat_id, to_db(title)),
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
[(chat_id, member_id) for member_id in members],
|
||||
)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise bad_request("Unknown member") from None
|
||||
return {**from_db(row), "member_ids": members}
|
||||
|
||||
|
||||
def get_chat(chat_id: UUID, user_id: UUID) -> dict:
|
||||
with connection() as conn:
|
||||
chat = conn.execute(
|
||||
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||
WHERE c.id = %s""",
|
||||
(user_id, chat_id),
|
||||
).fetchone()
|
||||
if chat is None:
|
||||
raise not_found("Chat not found")
|
||||
return from_db(chat)
|
||||
|
||||
|
||||
def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||
"""Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё»."""
|
||||
where, params = "", [user_id]
|
||||
if after is not None:
|
||||
where = "WHERE (c.created_at, c.id) > (%s, %s)"
|
||||
params += list(after)
|
||||
with connection() as conn:
|
||||
rows = conn.execute(
|
||||
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||
{where}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s""",
|
||||
(*params, limit + 1),
|
||||
).fetchall()
|
||||
return [from_db(row) for row in rows]
|
||||
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids"""
|
||||
|
||||
|
||||
def is_member(conn, chat_id: UUID, user_id: UUID) -> bool:
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||
).fetchone() is not None
|
||||
|
||||
|
||||
def check_member(chat_id: UUID, user_id: UUID) -> None:
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise not_found("Chat not found")
|
||||
|
||||
|
||||
def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID],
|
||||
idempotency_key: str, request_hash: bytes) -> dict:
|
||||
try:
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404
|
||||
raise not_found("Chat not found")
|
||||
if attachment_ids:
|
||||
# Только готовые вложения ЭТОГО чата. FOR SHARE: строку не удалят до конца транзакции.
|
||||
found = conn.execute(
|
||||
"""SELECT id FROM attachments
|
||||
WHERE id = ANY(%s) AND chat_id = %s AND state = 'ready' FOR SHARE""",
|
||||
(attachment_ids, chat_id),
|
||||
).fetchall()
|
||||
if len(found) != len(attachment_ids):
|
||||
raise not_found("Attachment not found")
|
||||
created = conn.execute(
|
||||
"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING
|
||||
RETURNING id, chat_id, sender_id, text, created_at""",
|
||||
(uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash),
|
||||
).fetchone()
|
||||
if created is not None:
|
||||
if attachment_ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
VALUES (%s, %s, %s, %s)""",
|
||||
[(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)],
|
||||
)
|
||||
return {**from_db(created), "attachment_ids": list(attachment_ids)}
|
||||
saved = conn.execute(
|
||||
f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m
|
||||
WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""",
|
||||
(chat_id, sender_id, idempotency_key),
|
||||
).fetchone()
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise not_found("Chat not found") from None
|
||||
if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен
|
||||
raise unavailable()
|
||||
if bytes(saved.pop("request_hash")) != request_hash:
|
||||
raise conflict("Idempotency-Key was already used with a different request")
|
||||
return from_db(saved)
|
||||
|
||||
|
||||
def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||
"""История чата после курсора (keyset-пагинация), до limit+1 строк."""
|
||||
where, params = "m.chat_id = %s", [chat_id]
|
||||
if after is not None:
|
||||
where += " AND (m.created_at, m.id) > (%s, %s)"
|
||||
params += list(after)
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise not_found("Chat not found")
|
||||
rows = conn.execute(
|
||||
f"""SELECT {MESSAGE_COLUMNS} FROM messages m
|
||||
WHERE {where}
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT %s""",
|
||||
(*params, limit + 1),
|
||||
).fetchall()
|
||||
return [from_db(row) for row in rows]
|
||||
|
||||
|
||||
# ---------- вложения ----------
|
||||
|
||||
ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"created_at, updated_at, object_key")
|
||||
|
||||
|
||||
def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str,
|
||||
size_bytes: int, sha256: str, width: int, height: int, object_key: str) -> None:
|
||||
"""Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в S3.
|
||||
|
||||
Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт
|
||||
и удалит его, если загрузка не дошла до ready (обрыв, падение реплики, отказ S3 или БД).
|
||||
"""
|
||||
try:
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
width, height, object_key, state)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""",
|
||||
(attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256,
|
||||
width, height, object_key),
|
||||
)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise not_found("Chat not found") from None
|
||||
|
||||
|
||||
def finish_upload(attachment_id: UUID) -> dict | None:
|
||||
"""Шаг 3 из 3: объект в S3 уже сохранён -> ready. None, если строку успел убрать janitor."""
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
f"""UPDATE attachments SET state = 'ready', updated_at = now()
|
||||
WHERE id = %s AND state = 'uploading'
|
||||
RETURNING {ATTACHMENT_COLUMNS}""",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
return from_db(row)
|
||||
|
||||
|
||||
def delete_upload_intent(attachment_id: UUID) -> None:
|
||||
with connection(wait_seconds=1) as conn:
|
||||
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,))
|
||||
|
||||
|
||||
def get_attachment(attachment_id: UUID, user_id: UUID) -> dict:
|
||||
"""Готовое вложение, если user_id — участник его чата. Иначе 404 (не раскрываем существование)."""
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))}
|
||||
FROM attachments a
|
||||
JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s
|
||||
WHERE a.id = %s AND a.state = 'ready'""",
|
||||
(user_id, attachment_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found("Attachment not found")
|
||||
return from_db(row)
|
||||
@@ -0,0 +1,40 @@
|
||||
# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт.
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def bad_request(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError:
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def not_found(message: str) -> ApiError:
|
||||
return ApiError(404, "not_found", message)
|
||||
|
||||
|
||||
def conflict(message: str) -> ApiError:
|
||||
return ApiError(409, "conflict", message)
|
||||
|
||||
|
||||
def unavailable(message: str = "Service temporarily unavailable") -> ApiError:
|
||||
return ApiError(503, "unavailable", message)
|
||||
|
||||
|
||||
def payload_too_large(message: str) -> ApiError:
|
||||
return ApiError(413, "payload_too_large", message)
|
||||
|
||||
|
||||
def unsupported_media_type(message: str) -> ApiError:
|
||||
return ApiError(415, "unsupported_media_type", message)
|
||||
@@ -0,0 +1,96 @@
|
||||
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
|
||||
# date: 2026-10-22 14:19:26, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_attachments.py
|
||||
# exit code: 0, 5.4 s
|
||||
|
||||
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
|
||||
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
|
||||
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
|
||||
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
|
||||
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
|
||||
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
|
||||
exactly 10485760 bytes (10 MiB) -> 201
|
||||
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
|
||||
PASS: the 10 MiB limit is exact
|
||||
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
|
||||
PASS: oversized streamed body is cut off, the API keeps working
|
||||
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 201 af872a80
|
||||
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
|
||||
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
|
||||
PASS: pixel limit is checked before decompression
|
||||
PNG cut in half (55908 bytes, image/png) -> 415
|
||||
PNG without IEND (111805 bytes, image/png) -> 415
|
||||
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
|
||||
PNG with garbage after IEND (111824 bytes, image/png) -> 415
|
||||
PNG signature only (8 bytes, image/png) -> 415
|
||||
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
|
||||
JPEG cut in half (746 bytes, image/jpeg) -> 415
|
||||
JPEG header only (20 bytes, image/jpeg) -> 415
|
||||
random bytes (1600 bytes, image/png) -> 415
|
||||
plain text (28 bytes, image/jpeg) -> 415
|
||||
empty body -> 400
|
||||
PASS: damaged and foreign files are rejected with 415, never 500
|
||||
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
|
||||
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
|
||||
empty PLTE (79 bytes, valid CRCs) -> 415
|
||||
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
|
||||
palette image without PLTE (67 bytes, valid CRCs) -> 415
|
||||
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
|
||||
PLTE after IDAT (85 bytes, valid CRCs) -> 415
|
||||
two PLTE chunks (103 bytes, valid CRCs) -> 415
|
||||
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
|
||||
tRNS before PLTE (98 bytes, valid CRCs) -> 415
|
||||
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
|
||||
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
|
||||
IEND with data (70 bytes, valid CRCs) -> 415
|
||||
unknown critical chunk (81 bytes, valid CRCs) -> 415
|
||||
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
|
||||
gAMA after IDAT (85 bytes, valid CRCs) -> 415
|
||||
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
|
||||
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
|
||||
private chunk after IDAT (181 bytes, valid CRCs) -> 415
|
||||
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
|
||||
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
|
||||
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
|
||||
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
|
||||
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
|
||||
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
|
||||
palette depth 2 with 4 entries -> 201, downloaded byte for byte
|
||||
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
|
||||
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
|
||||
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
|
||||
PNG bytes sent as image/jpeg -> 415
|
||||
JPEG bytes sent as image/png -> 415
|
||||
GIF bytes sent as image/png -> 415
|
||||
PNG sent as image/gif -> 415
|
||||
PNG sent as text/html -> 415
|
||||
PNG sent as application/octet-stream -> 415
|
||||
PNG sent as multipart/form-data -> 415
|
||||
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
|
||||
PASS: the type is taken from the content; a declared type that does not match is refused
|
||||
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
|
||||
no filename -> 400
|
||||
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
|
||||
PASS: filename never becomes a path; dangerous names are refused
|
||||
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
|
||||
upload into someone else's / unknown chat -> 404, without token -> 401
|
||||
variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200
|
||||
PASS: only chat members can see an attachment
|
||||
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
|
||||
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
|
||||
attachment of another chat (author is a member of both) -> 404
|
||||
own + foreign attachment -> 404
|
||||
unknown attachment_id -> 404
|
||||
outsider referencing an attachment of the chat -> 404
|
||||
duplicate ids -> 400
|
||||
11 ids -> 400
|
||||
bad UUID -> 400
|
||||
empty text without attachments -> 400
|
||||
whitespace text without attachments -> 400
|
||||
PASS: only attachments of the same chat can be linked; rejected requests change nothing
|
||||
same key + same attachments -> same 201; other attachments -> 409; missing field == []
|
||||
PASS: Idempotency-Key covers attachment_ids
|
||||
sent 1.1 MB of a declared 5 MB body and closed the connection
|
||||
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
|
||||
PASS: an interrupted upload is dropped
|
||||
12 passed, 0 failed
|
||||
@@ -0,0 +1,18 @@
|
||||
# balancing: balancing, shared sessions/data/files, X-Instance-Id
|
||||
# date: 2026-10-22 14:22:27, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_balancing.py
|
||||
# exit code: 0, 2.0 s
|
||||
|
||||
20 x GET /users/me with one token -> replicas: {'d590adc21943': 10, '343119df2488': 10}
|
||||
PASS: one session works on every replica, no second login (sessions are shared in Redis)
|
||||
message written by 343119df2488; read back by ['343119df2488', 'd590adc21943']
|
||||
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
|
||||
logout on d590adc21943; old token then rejected by ['343119df2488', 'd590adc21943']
|
||||
PASS: logout on one replica revokes the session on all of them
|
||||
first POST on d590adc21943 -> 201 6c2f624d-c728-4738-b486-b17ab735c0ae; same key on 343119df2488 -> 201 6c2f624d-c728-4738-b486-b17ab735c0ae
|
||||
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
|
||||
file uploaded via d590adc21943; downloaded byte for byte via ['343119df2488', 'd590adc21943']
|
||||
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
|
||||
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['343119df2488', 'd590adc21943']; api containers: ['343119df2488', 'd590adc21943']
|
||||
PASS: X-Instance-Id is in every response and points to a real container
|
||||
6 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# check: offline contract check (make check)
|
||||
# date: 2026-10-22 14:19:16, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe scripts/check_contract.py
|
||||
# exit code: 0, 0.1 s
|
||||
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,47 @@
|
||||
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
|
||||
# date: 2026-10-22 14:21:48, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_consistency.py
|
||||
# exit code: 0, 133.0 s
|
||||
|
||||
declared 9 000 000 bytes, sent 211817, closed the connection
|
||||
declared 9 000 000 bytes, sent 3111817, closed the connection
|
||||
rows in attachments: 448 -> 448; objects in bucket: 444 -> 444
|
||||
temporary upload files in api containers: {'343119df2488': 0, 'd590adc21943': 0} (removed when the request ends)
|
||||
api containers have no volumes: the original is never stored locally
|
||||
PASS: an interrupted upload leaves no row, no object and no local file
|
||||
crash after S3 PUT: row 9aa0bc8a state=uploading, object originals/8ae03455b55a4f798b7bc6248b450ef9 exists in S3
|
||||
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
|
||||
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 446, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.133} -> fresh row and object are kept (the upload might still be running)
|
||||
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "objects_scanned": 446, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.184} -> nothing deleted
|
||||
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 445, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.132} -> row and object deleted
|
||||
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 445, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.179} -> idempotent
|
||||
the ready attachment of the same chat is untouched
|
||||
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
|
||||
object originals/101748e3a911439a9c1e11e768c99822 put into the bucket without a row in PostgreSQL
|
||||
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 447, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.191} -> fresh object kept
|
||||
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 447, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 4, "seconds": 0.188} -> reported, kept
|
||||
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 447, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 4, "seconds": 0.1} -> orphan deleted
|
||||
ready attachments older than the grace period are untouched (they have rows)
|
||||
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=5 and keeps the row; download -> 503 (not 500)
|
||||
PASS: orphans are found and deleted only after the grace period; missing objects are reported
|
||||
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
|
||||
upload -> 503 unavailable in 19453 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose start s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 201
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 448, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 5, "seconds": 0.191}
|
||||
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
--- S3 frozen (no answer, timeouts): docker compose pause s3
|
||||
upload -> 503 unavailable in 14891 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose unpause s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 201
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 450, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 5, "seconds": 0.115}
|
||||
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
5 passed, 0 failed
|
||||
@@ -0,0 +1,21 @@
|
||||
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
|
||||
# date: 2026-10-22 14:22:40, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_db.py
|
||||
# exit code: 0, 6.2 s
|
||||
|
||||
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 335 -> 336
|
||||
PASS: chat row was inserted and then rolled back together with members
|
||||
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
|
||||
PASS: the database itself rejects a message from a non-member
|
||||
stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97)
|
||||
PASS: password is stored as Argon2id hash
|
||||
unique constraint on users: UNIQUE (username)
|
||||
PASS: username is unique in the database
|
||||
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; running the migrate service once more
|
||||
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; log: Migrations applied: none (schema is up to date)
|
||||
PASS: migrations are applied once, a repeated run does nothing
|
||||
api replicas that checked the schema at start: 14; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago
|
||||
PASS: only the migrate service applies migrations, replicas wait for the schema
|
||||
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
|
||||
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
|
||||
7 passed, 0 failed
|
||||
@@ -0,0 +1,104 @@
|
||||
=== Data volume
|
||||
messages_total | messages_in_big_chat
|
||||
----------------+----------------------
|
||||
35618 | 15000
|
||||
|
||||
|
||||
=== History index
|
||||
indexdef
|
||||
----------------------------------------------------------------------------------------
|
||||
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
|
||||
|
||||
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.006..0.010 rows=51 loops=1)
|
||||
Buffers: shared hit=4
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1)
|
||||
Index Cond: (chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid)
|
||||
Buffers: shared hit=4
|
||||
Planning:
|
||||
Buffers: shared hit=8
|
||||
Planning Time: 0.071 ms
|
||||
Execution Time: 0.020 ms
|
||||
|
||||
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.006..0.011 rows=51 loops=1)
|
||||
Buffers: shared hit=5
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '4bd22867-547f-4ecf-8182-443fbb004188'::uuid)))
|
||||
Buffers: shared hit=5
|
||||
Planning Time: 0.040 ms
|
||||
Execution Time: 0.029 ms
|
||||
|
||||
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.019..0.208 rows=51 loops=1)
|
||||
Buffers: shared hit=59
|
||||
-> Index Scan using messages_history on messages m (actual time=0.019..0.204 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '4bd22867-547f-4ecf-8182-443fbb004188'::uuid)))
|
||||
Buffers: shared hit=59
|
||||
SubPlan 1
|
||||
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
|
||||
Sort Key: a."position"
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Recheck Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Index Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
Planning:
|
||||
Buffers: shared hit=63
|
||||
Planning Time: 0.156 ms
|
||||
Execution Time: 0.225 ms
|
||||
|
||||
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
QUERY PLAN
|
||||
--------------------------------------------------------------------------------
|
||||
Limit (actual time=3.056..3.060 rows=51 loops=1)
|
||||
Buffers: shared hit=585
|
||||
-> Sort (actual time=2.647..2.857 rows=10051 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: quicksort Memory: 2377kB
|
||||
Buffers: shared hit=585
|
||||
-> Seq Scan on messages (actual time=0.005..1.278 rows=15000 loops=1)
|
||||
Filter: (chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid)
|
||||
Rows Removed by Filter: 20618
|
||||
Buffers: shared hit=585
|
||||
Planning Time: 0.033 ms
|
||||
Execution Time: 3.107 ms
|
||||
|
||||
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
QUERY PLAN
|
||||
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=1.432..1.436 rows=51 loops=1)
|
||||
Buffers: shared hit=585
|
||||
-> Sort (actual time=1.432..1.433 rows=51 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 31kB
|
||||
Buffers: shared hit=585
|
||||
-> Seq Scan on messages (actual time=0.678..1.116 rows=5000 loops=1)
|
||||
Filter: ((chat_id = '872415a3-3ab7-4e2b-bbc5-e5d03904fd1d'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '4bd22867-547f-4ecf-8182-443fbb004188'::uuid)))
|
||||
Rows Removed by Filter: 30618
|
||||
Buffers: shared hit=585
|
||||
Planning:
|
||||
Buffers: shared hit=4
|
||||
Planning Time: 0.049 ms
|
||||
Execution Time: 1.450 ms
|
||||
|
||||
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
explain_users_left
|
||||
--------------------
|
||||
0
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
|
||||
# date: 2026-10-22 14:22:45, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/explain.py
|
||||
# exit code: 0, 1.1 s
|
||||
|
||||
=== Data volume
|
||||
=== History index
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
Buffers: shared hit=4
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1)
|
||||
Buffers: shared hit=4
|
||||
Buffers: shared hit=8
|
||||
Execution Time: 0.020 ms
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
Buffers: shared hit=5
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1)
|
||||
Buffers: shared hit=5
|
||||
Execution Time: 0.029 ms
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
Buffers: shared hit=59
|
||||
-> Index Scan using messages_history on messages m (actual time=0.019..0.204 rows=51 loops=1)
|
||||
Buffers: shared hit=59
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
Buffers: shared hit=63
|
||||
Execution Time: 0.225 ms
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
Buffers: shared hit=585
|
||||
Buffers: shared hit=585
|
||||
-> Seq Scan on messages (actual time=0.005..1.278 rows=15000 loops=1)
|
||||
Buffers: shared hit=585
|
||||
Execution Time: 3.107 ms
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
Buffers: shared hit=585
|
||||
Buffers: shared hit=585
|
||||
-> Seq Scan on messages (actual time=0.678..1.116 rows=5000 loops=1)
|
||||
Buffers: shared hit=585
|
||||
Buffers: shared hit=4
|
||||
Execution Time: 1.450 ms
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
PASS: full plans saved to evidence\explain-history.txt
|
||||
@@ -0,0 +1,33 @@
|
||||
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
|
||||
# date: 2026-10-22 14:23:28, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/run_failover.py
|
||||
# exit code: 0, 43.2 s
|
||||
|
||||
hook: scripts\stop-one.cmd; api replicas: ['343119df2488', 'd590adc21943']
|
||||
=== run 1: api containers before
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api 343119df2488 Up 27 seconds (healthy)
|
||||
api d590adc21943 Up 27 seconds (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
|
||||
{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.422}␍
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api 343119df2488 Exited (0) 11 seconds ago
|
||||
api d590adc21943 Up 43 seconds (healthy)
|
||||
the hook stopped 343119df2488 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start 343119df2488 -> healthy and back in HAProxy
|
||||
=== run 2: api containers before
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api 343119df2488 Up 4 seconds (healthy)
|
||||
api d590adc21943 Up 48 seconds (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
|
||||
{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 4.531}␍
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api 343119df2488 Up 20 seconds (healthy)
|
||||
api d590adc21943 Exited (0) 11 seconds ago
|
||||
the hook stopped d590adc21943 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start d590adc21943 -> healthy and back in HAProxy
|
||||
PASS: public failover test passed 2 times; every api replica was stopped once and restored
|
||||
@@ -0,0 +1,29 @@
|
||||
# failures: Redis and PostgreSQL outages via the balancer
|
||||
# date: 2026-10-22 14:24:54, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_failures.py
|
||||
# exit code: 0, 55.6 s
|
||||
|
||||
stopping redis
|
||||
/health/live -> 200, /health/ready -> 503 (8000 ms)
|
||||
valid token: GET /users/me -> 503 unavailable (8031 ms), session cannot be checked
|
||||
unknown token -> 503, not 200: nobody is trusted without a check
|
||||
no token -> 401
|
||||
login -> 503: there is nowhere to store the session
|
||||
registration -> 201: it needs only PostgreSQL
|
||||
starting redis
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: Redis outage gives 503 and does not let anyone in without a session check
|
||||
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
|
||||
right after the crash: GET /users/me -> 503
|
||||
starting redis
|
||||
after start: the session issued before the crash still works (AOF, appendfsync always)
|
||||
PASS: sessions survive a Redis crash, not only a clean restart
|
||||
stopping postgres
|
||||
/health/live -> 200, /health/ready -> 503 (1031 ms)
|
||||
valid token: users/me, chats, history, new message -> 503 unavailable, no details
|
||||
no token -> 401, unknown token -> 401: anonymous users are not let in
|
||||
login -> 503, registration -> 503
|
||||
starting postgres
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
|
||||
3 passed, 0 failed
|
||||
@@ -0,0 +1,22 @@
|
||||
# idempotency: Idempotency-Key, parallel same key via replicas
|
||||
# date: 2026-10-22 14:22:30, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_idempotency.py
|
||||
# exit code: 0, 2.5 s
|
||||
|
||||
first -> 201; same JSON written differently -> 201; plain retry -> 201
|
||||
PASS: a retry with the same key returns the original 201 and message, no duplicate
|
||||
20 parallel POSTs with one key: statuses {201: 20}, replicas {'343119df2488': 10, 'd590adc21943': 10}, different bodies in answers: 1, messages in history: 1
|
||||
PASS: parallel requests with one key through different replicas create exactly one message
|
||||
20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A']
|
||||
PASS: the first body wins, the same body gets 201, the other body gets 409
|
||||
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
|
||||
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
|
||||
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
|
||||
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
|
||||
key used by a 400 and by a 404 first, then valid request -> 201
|
||||
PASS: rejected requests (400, 404) do not store a result for the key
|
||||
same key and body by an outsider -> 404; by alice after logout -> 401
|
||||
PASS: a retry checks the session and chat access again, the saved answer is not leaked
|
||||
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
|
||||
PASS: an unconfirmed POST can be safely retried with the same key
|
||||
8 passed, 0 failed
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 4,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.047,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 7191,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 239.325,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.0,
|
||||
"p95": 32.0,
|
||||
"p99": 32.0
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# load: public load generator, 30 s, concurrency 4
|
||||
# date: 2026-10-22 14:23:58, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||
# exit code: 0, 30.3 s
|
||||
|
||||
{
|
||||
"lab": 4,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.047,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 7191,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 239.325,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.0,
|
||||
"p95": 32.0,
|
||||
"p99": 32.0
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence)
|
||||
# date: 2026-10-22 14:25:42, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/run_persistence.py
|
||||
# exit code: 0, 47.9 s
|
||||
|
||||
before: container, ID, started at
|
||||
messenger-lab4-api-1 343119df2488 2026-10-22T11:23:03.387600262Z
|
||||
messenger-lab4-api-2 d590adc21943 2026-10-22T11:23:24.689368662Z
|
||||
messenger-lab4-janitor-1 d50ac08018c1 2026-10-22T11:18:17.423165415Z
|
||||
messenger-lab4-lb-1 0d5b262b09b3 2026-10-22T11:03:30.971491955Z
|
||||
messenger-lab4-postgres-1 84794c3ac2c5 2026-10-22T11:24:52.672662576Z
|
||||
messenger-lab4-redis-1 3130fd5532b0 2026-10-22T11:24:32.055032931Z
|
||||
messenger-lab4-s3-1 122ad603e3be 2026-10-22T11:20:52.20174809Z
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.␍
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
after (hook + waiting for readiness took 47.0 s): container, ID, started at, restarted?
|
||||
messenger-lab4-api-1 343119df2488 2026-10-22T11:25:36.201352227Z same container: True, restarted: True
|
||||
messenger-lab4-api-2 d590adc21943 2026-10-22T11:25:36.073058122Z same container: True, restarted: True
|
||||
messenger-lab4-janitor-1 d50ac08018c1 2026-10-22T11:25:32.252887377Z same container: True, restarted: True
|
||||
messenger-lab4-lb-1 0d5b262b09b3 2026-10-22T11:25:40.174401907Z same container: True, restarted: True
|
||||
messenger-lab4-postgres-1 84794c3ac2c5 2026-10-22T11:24:56.311470336Z same container: True, restarted: True
|
||||
messenger-lab4-redis-1 3130fd5532b0 2026-10-22T11:24:56.150164333Z same container: True, restarted: True
|
||||
messenger-lab4-s3-1 122ad603e3be 2026-10-22T11:25:06.0234901Z same container: True, restarted: True
|
||||
PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, new start time); data, the attachment bytes and the old token survived
|
||||
@@ -0,0 +1,10 @@
|
||||
# race: lab2 races via the balancer
|
||||
# date: 2026-10-22 14:22:34, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_race.py
|
||||
# exit code: 0, 1.1 s
|
||||
|
||||
sent 100 messages from 10 clients in 0.19 s; read 100 in 4 pages, unique ids: 100
|
||||
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
|
||||
20 parallel registrations of one username: 201 x1, 409 x19
|
||||
PASS: concurrent registration created exactly one user
|
||||
2 passed, 0 failed
|
||||
@@ -0,0 +1,14 @@
|
||||
# redis: Redis as the session store
|
||||
# date: 2026-10-22 14:22:43, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/redis_info.py
|
||||
# exit code: 0, 2.9 s
|
||||
|
||||
session key: session:e57bdd083b4cb87e...; value is the user id: True
|
||||
TTL of the key, seconds: 3599
|
||||
sessions in Redis: 330
|
||||
config appendonly = yes
|
||||
config appendfsync = always
|
||||
config maxmemory-policy = noeviction
|
||||
aof_enabled:1
|
||||
aof_last_write_status:ok
|
||||
without password: NOAUTH Authentication required.
|
||||
@@ -0,0 +1,44 @@
|
||||
# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap
|
||||
# date: 2026-10-22 14:19:35, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_s3_security.py
|
||||
# exit code: 0, 9.6 s
|
||||
|
||||
anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied
|
||||
anonymous head_object directly to S3 (from the data network) -> 403
|
||||
anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied
|
||||
anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied
|
||||
anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied
|
||||
the same file through the API for a chat member -> 200
|
||||
PASS: the bucket is private: no anonymous read, list, write or policy read
|
||||
API key: list buckets -> ['messenger-attachments']
|
||||
API key: get own object -> allowed
|
||||
API key: put object -> allowed
|
||||
API key: delete object -> allowed
|
||||
API key: list bucket -> allowed
|
||||
API key: create another bucket -> denied: AccessDenied
|
||||
API key: make bucket public (put policy) -> denied: NotImplemented
|
||||
API key: read bucket policy -> denied: NoSuchBucketPolicy
|
||||
API key: delete bucket -> denied: AccessDenied
|
||||
API key: change versioning -> denied: NotImplemented
|
||||
PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied
|
||||
compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand)
|
||||
127.0.0.1:8333 from the host -> connection refused
|
||||
127.0.0.1:9333 from the host -> connection refused
|
||||
127.0.0.1:8888 from the host -> connection refused
|
||||
127.0.0.1:19333 from the host -> connection refused
|
||||
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of lb: -
|
||||
secrets of migrate: ['database_url']
|
||||
secrets of postgres: ['postgres_password']
|
||||
secrets of redis: ['redis_auth']
|
||||
secrets of s3: ['s3_config']
|
||||
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
|
||||
PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init
|
||||
s3-init: s3-init: bucket messenger-attachments exists
|
||||
s3-init: s3-init: api key can put/get/list/delete objects of its bucket
|
||||
s3-init: s3-init: api key cannot create buckets (AccessDenied)
|
||||
s3-init: s3-init: bucket is private (anonymous list/get/put -> [403])
|
||||
after the repeated bootstrap: old file downloads, new upload works
|
||||
PASS: bucket/user/policy bootstrap can be repeated safely
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# smoke: public smoke via the balancer (make test), with attachment
|
||||
# date: 2026-10-22 14:19:20, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
# exit code: 0, 0.8 s
|
||||
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,40 @@
|
||||
# stack: stand layout: replicas, ports, limits, networks, secrets, S3
|
||||
# date: 2026-10-22 14:19:19, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/check_stack.py
|
||||
# exit code: 0, 2.9 s
|
||||
|
||||
host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1
|
||||
Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop
|
||||
|
||||
SERVICE CONTAINER ID STATUS PORTS
|
||||
api 343119df2488 Up 17 seconds (healthy) 8080/tcp
|
||||
api d590adc21943 Up 18 seconds (healthy) 8080/tcp
|
||||
janitor d50ac08018c1 Up About a minute
|
||||
lb 0d5b262b09b3 Up 15 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
migrate 6c3123ccf237 Exited (0) About a minute ago
|
||||
postgres 84794c3ac2c5 Up 16 minutes (healthy)
|
||||
redis 3130fd5532b0 Up 16 minutes (healthy)
|
||||
s3 122ad603e3be Up 16 minutes (healthy)
|
||||
s3-init 47c2d7d47ed2 Exited (0) About a minute ago
|
||||
|
||||
container image cpus memory host ports networks
|
||||
messenger-lab4-api-1 messenger-lab4-api 1.00 512 MiB - data,edge
|
||||
messenger-lab4-api-2 messenger-lab4-api 1.00 512 MiB - data,edge
|
||||
messenger-lab4-janitor-1 messenger-lab4-janitor 0.25 192 MiB - data
|
||||
messenger-lab4-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge
|
||||
messenger-lab4-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data
|
||||
messenger-lab4-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data
|
||||
messenger-lab4-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data
|
||||
|
||||
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of lb: -
|
||||
secrets of migrate: ['database_url']
|
||||
secrets of postgres: ['postgres_password']
|
||||
secrets of redis: ['redis_auth']
|
||||
secrets of s3: ['s3_config']
|
||||
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
|
||||
|
||||
through http://127.0.0.1:8080 one session is served by 2 replicas: ['343119df2488', 'd590adc21943']
|
||||
/health/ready -> 200 from d590adc21943
|
||||
PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor only in the internal data network, S3 admin keys only in s3 and s3-init
|
||||
@@ -0,0 +1,26 @@
|
||||
# student-tests: lab1/lab2 API tests via the balancer
|
||||
# date: 2026-10-22 14:22:33, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/test_api.py
|
||||
# exit code: 0, 2.9 s
|
||||
|
||||
PASS: duplicate username returns 409
|
||||
PASS: invalid JSON returns 400
|
||||
PASS: wrong and extra fields return 400
|
||||
PASS: foreign chat returns 404
|
||||
PASS: empty and too long messages return 400, 4000 chars is ok
|
||||
PASS: pagination works across multiple pages, foreign cursor returns 400
|
||||
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
|
||||
PASS: chat list pagination
|
||||
PASS: unicode text is stored as is
|
||||
PASS: strings with U+0000 are stored and returned as is
|
||||
PASS: user response has no password or hash
|
||||
PASS: password length 12..128, blank password is rejected
|
||||
PASS: Basic login returns opaque Bearer token with expiry
|
||||
PASS: wrong password and unknown user both return the same 401
|
||||
PASS: malformed Basic header returns 401
|
||||
PASS: password with colon and unicode works
|
||||
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
|
||||
PASS: X-User-Id does not change the current user
|
||||
PASS: 401 comes before 400 and 404
|
||||
PASS: logout revokes only the current session
|
||||
20 passed, 0 failed
|
||||
@@ -0,0 +1,57 @@
|
||||
# Сводка проверок
|
||||
|
||||
| Шаг | Что проверяет | Результат | Время, с | Файл |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| check | offline contract check (make check) | PASS | 0 | evidence/check.txt |
|
||||
| stack | stand layout: replicas, ports, limits, networks, secrets, S3 | PASS | 3 | evidence/stack.txt |
|
||||
| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/smoke.txt |
|
||||
| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 5 | evidence/attachments.txt |
|
||||
| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt |
|
||||
| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 133 | evidence/consistency.txt |
|
||||
| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 37 | evidence/upload-memory.txt |
|
||||
| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt |
|
||||
| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt |
|
||||
| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt |
|
||||
| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt |
|
||||
| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 6 | evidence/db.txt |
|
||||
| redis | Redis as the session store | PASS | 3 | evidence/redis.txt |
|
||||
| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt |
|
||||
| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 43 | evidence/failover.txt |
|
||||
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt |
|
||||
| failures | Redis and PostgreSQL outages via the balancer | PASS | 56 | evidence/failures.txt |
|
||||
| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 48 | evidence/persistence.txt |
|
||||
|
||||
# Замеры
|
||||
|
||||
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-22, commit c942c61 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB.
|
||||
|
||||
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
|
||||
|
||||
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 1 | 343119df2488 | 2 | 123 | 0 | 4.422 |
|
||||
| 2 | d590adc21943 | 2 | 124 | 0 | 4.531 |
|
||||
|
||||
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
|
||||
|
||||
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 2.87 / 3.52 | 343119df2488: 5 | 94.8 | 107.6 | 12.8 | 136.0 |
|
||||
| | | | | d590adc21943: 5 | 91.2 | 104.5 | 13.3 | 136.2 |
|
||||
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 343119df2488: 5 | 95.1 | 96.6 | 1.5 | 86.5 |
|
||||
| | | | | d590adc21943: 5 | 92.1 | 96.6 | 4.6 | 83.5 |
|
||||
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.5 / 1.78 | 343119df2488: 10 | 93.2 | 116.2 | 23.0 | 158.2 |
|
||||
| | | | | d590adc21943: 10 | 95.0 | 118.6 | 23.6 | 167.8 |
|
||||
|
||||
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
|
||||
|
||||
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.325 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.422, 4.531 с (hook ждёт остановки ~4 с); потерь нет |
|
||||
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 47.0 с до готовности; потерь нет |
|
||||
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19453 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
|
||||
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14891 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
|
||||
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
|
||||
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"date": "2026-10-22 14:22",
|
||||
"api_limit_mib": 512,
|
||||
"api_limit_cpus": 1.0,
|
||||
"runs": [
|
||||
{
|
||||
"scenario": "10 x ~9.5 MiB PNG (noise)",
|
||||
"concurrency": 10,
|
||||
"file_bytes": [
|
||||
9953047
|
||||
],
|
||||
"total_mib": 94.9,
|
||||
"all_201": true,
|
||||
"wall_seconds": 3.52,
|
||||
"latency_s": {
|
||||
"p50": 2.87,
|
||||
"max": 3.52
|
||||
},
|
||||
"sampler_rounds": 18,
|
||||
"replicas": [
|
||||
{
|
||||
"replica": "343119df2488",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 94.8,
|
||||
"rss_peak_mib": 107.6,
|
||||
"rss_growth_mib": 12.8,
|
||||
"rss_peak_sampled_mib": 107.6,
|
||||
"cgroup_peak_mib": 136.0,
|
||||
"cgroup_peak_sampled_mib": 134.5
|
||||
},
|
||||
{
|
||||
"replica": "d590adc21943",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 91.2,
|
||||
"rss_peak_mib": 104.5,
|
||||
"rss_growth_mib": 13.3,
|
||||
"rss_peak_sampled_mib": 104.5,
|
||||
"cgroup_peak_mib": 136.2,
|
||||
"cgroup_peak_sampled_mib": 133.3
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)",
|
||||
"concurrency": 10,
|
||||
"file_bytes": [
|
||||
116714
|
||||
],
|
||||
"total_mib": 1.1,
|
||||
"all_201": true,
|
||||
"wall_seconds": 1.09,
|
||||
"latency_s": {
|
||||
"p50": 0.85,
|
||||
"max": 1.09
|
||||
},
|
||||
"sampler_rounds": 12,
|
||||
"replicas": [
|
||||
{
|
||||
"replica": "343119df2488",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 95.1,
|
||||
"rss_peak_mib": 96.6,
|
||||
"rss_growth_mib": 1.5,
|
||||
"rss_peak_sampled_mib": 96.6,
|
||||
"cgroup_peak_mib": 86.5,
|
||||
"cgroup_peak_sampled_mib": 77.6
|
||||
},
|
||||
{
|
||||
"replica": "d590adc21943",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 92.1,
|
||||
"rss_peak_mib": 96.6,
|
||||
"rss_growth_mib": 4.6,
|
||||
"rss_peak_sampled_mib": 96.6,
|
||||
"cgroup_peak_mib": 83.5,
|
||||
"cgroup_peak_sampled_mib": 77.8
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"scenario": "20 mixed: large + 40 Mpx + JPEG",
|
||||
"concurrency": 20,
|
||||
"file_bytes": [
|
||||
1492,
|
||||
116714,
|
||||
9953047
|
||||
],
|
||||
"total_mib": 133.2,
|
||||
"all_201": true,
|
||||
"wall_seconds": 1.78,
|
||||
"latency_s": {
|
||||
"p50": 1.5,
|
||||
"max": 1.78
|
||||
},
|
||||
"sampler_rounds": 12,
|
||||
"replicas": [
|
||||
{
|
||||
"replica": "343119df2488",
|
||||
"uploads_served": 10,
|
||||
"rss_before_mib": 93.2,
|
||||
"rss_peak_mib": 116.2,
|
||||
"rss_growth_mib": 23.0,
|
||||
"rss_peak_sampled_mib": 116.2,
|
||||
"cgroup_peak_mib": 158.2,
|
||||
"cgroup_peak_sampled_mib": 146.7
|
||||
},
|
||||
{
|
||||
"replica": "d590adc21943",
|
||||
"uploads_served": 10,
|
||||
"rss_before_mib": 95.0,
|
||||
"rss_peak_mib": 118.6,
|
||||
"rss_growth_mib": 23.6,
|
||||
"rss_peak_sampled_mib": 118.6,
|
||||
"cgroup_peak_mib": 167.8,
|
||||
"cgroup_peak_sampled_mib": 167.7
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"single_file_extra_rss_mib": {
|
||||
"naive": 161,
|
||||
"streaming": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas
|
||||
# date: 2026-10-22 14:22:25, commit: c942c61 + uncommitted changes
|
||||
# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json
|
||||
# exit code: 0, 37.0 s
|
||||
|
||||
client: Python 3.11.1 on Windows 10
|
||||
preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...
|
||||
large: 9953047 bytes; 40 Mpx: 116714 bytes
|
||||
{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_201": true, "wall_seconds": 3.52, "latency_s": {"p50": 2.87, "max": 3.52}, "sampler_rounds": 18, "replicas": [{"replica": "343119df2488", "uploads_served": 5, "rss_before_mib": 94.8, "rss_peak_mib": 107.6, "rss_growth_mib": 12.8, "rss_peak_sampled_mib": 107.6, "cgroup_peak_mib": 136.0, "cgroup_peak_sampled_mib": 134.5}, {"replica": "d590adc21943", "uploads_served": 5, "rss_before_mib": 91.2, "rss_peak_mib": 104.5, "rss_growth_mib": 13.3, "rss_peak_sampled_mib": 104.5, "cgroup_peak_mib": 136.2, "cgroup_peak_sampled_mib": 133.3}]}
|
||||
{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_201": true, "wall_seconds": 1.09, "latency_s": {"p50": 0.85, "max": 1.09}, "sampler_rounds": 12, "replicas": [{"replica": "343119df2488", "uploads_served": 5, "rss_before_mib": 95.1, "rss_peak_mib": 96.6, "rss_growth_mib": 1.5, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 86.5, "cgroup_peak_sampled_mib": 77.6}, {"replica": "d590adc21943", "uploads_served": 5, "rss_before_mib": 92.1, "rss_peak_mib": 96.6, "rss_growth_mib": 4.6, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 83.5, "cgroup_peak_sampled_mib": 77.8}]}
|
||||
{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_201": true, "wall_seconds": 1.78, "latency_s": {"p50": 1.5, "max": 1.78}, "sampler_rounds": 12, "replicas": [{"replica": "343119df2488", "uploads_served": 10, "rss_before_mib": 93.2, "rss_peak_mib": 116.2, "rss_growth_mib": 23.0, "rss_peak_sampled_mib": 116.2, "cgroup_peak_mib": 158.2, "cgroup_peak_sampled_mib": 146.7}, {"replica": "d590adc21943", "uploads_served": 10, "rss_before_mib": 95.0, "rss_peak_mib": 118.6, "rss_growth_mib": 23.6, "rss_peak_sampled_mib": 118.6, "cgroup_peak_mib": 167.8, "cgroup_peak_sampled_mib": 167.7}]}
|
||||
one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB
|
||||
PASS: all uploads succeeded; largest RSS growth of one replica: 23.6 MiB (limit 512 MiB); results in evidence/upload-memory.json
|
||||
@@ -0,0 +1,77 @@
|
||||
# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики).
|
||||
# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить
|
||||
# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky).
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
# При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с.
|
||||
hard-stop-after 10s
|
||||
|
||||
# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик.
|
||||
# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска.
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s # перечитывать список реплик раз в 2 с
|
||||
hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с
|
||||
hold nx 2s
|
||||
hold other 2s
|
||||
hold refused 2s
|
||||
hold timeout 2s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlog-normal # в лог только ошибки, а не каждый запрос
|
||||
timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос
|
||||
timeout client 30s
|
||||
# Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка
|
||||
# при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт,
|
||||
# сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT).
|
||||
timeout server 30s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 5s
|
||||
timeout check 1s
|
||||
# Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода.
|
||||
# Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key.
|
||||
retries 2
|
||||
option redispatch 1
|
||||
|
||||
frontend api_in
|
||||
bind :8080
|
||||
option forwardfor
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
# Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
|
||||
http-check expect status 200
|
||||
# До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT.
|
||||
# Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP.
|
||||
# observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки.
|
||||
server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none
|
||||
# Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis
|
||||
# или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате
|
||||
# контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает.
|
||||
# Пока готова хоть одна реплика, эти записи не используются.
|
||||
# allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend).
|
||||
server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
|
||||
option allbackups
|
||||
|
||||
# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1).
|
||||
# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда.
|
||||
frontend stats
|
||||
bind :8404
|
||||
monitor-uri /lb-health
|
||||
monitor fail if { nbsrv(api) lt 1 }
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
@@ -0,0 +1,13 @@
|
||||
bind 0.0.0.0
|
||||
port 6379
|
||||
protected-mode yes
|
||||
dir /data
|
||||
|
||||
appendonly yes
|
||||
appendfsync always
|
||||
save ""
|
||||
|
||||
maxmemory 64mb
|
||||
maxmemory-policy noeviction
|
||||
|
||||
include /run/secrets/redis_auth
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/bin/sh
|
||||
# Bootstrap of the S3 storage (MinIO), run by the one-shot service s3-init before api starts.
|
||||
# Idempotent: every step can be repeated (docker compose run --rm s3-init) without losing data.
|
||||
# 1. private bucket S3_BUCKET (no anonymous access, versioning off);
|
||||
# 2. policy messenger-api: only Get/Put/Delete/List objects of this bucket (JSON below);
|
||||
# 3. separate user for api/janitor with this policy. Root credentials stay in s3 and s3-init only.
|
||||
# Secrets are read from /run/secrets and are never printed.
|
||||
set -eu
|
||||
|
||||
BUCKET="${S3_BUCKET:?S3_BUCKET is not set}"
|
||||
ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
API_KEY="$(cat /run/secrets/s3_access_key_id)"
|
||||
API_SECRET="$(cat /run/secrets/s3_secret_access_key)"
|
||||
POLICY=messenger-api
|
||||
export MC_CONFIG_DIR=/tmp/mc
|
||||
|
||||
# Least privilege for the api key: objects of ONE bucket. No bucket admin, policies, users or other buckets.
|
||||
cat > /tmp/policy.json <<POLICY_JSON
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::${BUCKET}"]},
|
||||
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload"],
|
||||
"Resource": ["arn:aws:s3:::${BUCKET}/*"]}
|
||||
]
|
||||
}
|
||||
POLICY_JSON
|
||||
|
||||
i=0
|
||||
until mc alias set local "${S3_ENDPOINT:-http://s3:9000}" "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -ge 30 ] && { echo "s3-init: MinIO is not reachable"; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
|
||||
mc mb --ignore-existing "local/${BUCKET}" >/dev/null
|
||||
mc anonymous set none "local/${BUCKET}" >/dev/null
|
||||
echo "s3-init: bucket ${BUCKET} exists"
|
||||
|
||||
mc admin policy create local "$POLICY" /tmp/policy.json >/dev/null
|
||||
echo "s3-init: policy ${POLICY} created/updated"
|
||||
|
||||
mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null
|
||||
info="$(mc admin user info local "$API_KEY" 2>/dev/null || true)"
|
||||
case "$info" in
|
||||
*"$POLICY"*) ;;
|
||||
*) mc admin policy attach local "$POLICY" --user "$API_KEY" >/dev/null ;;
|
||||
esac
|
||||
echo "s3-init: api user exists with policy ${POLICY} only"
|
||||
|
||||
access="$(mc anonymous get "local/${BUCKET}")"
|
||||
case "$access" in
|
||||
*private*) echo "s3-init: bucket is private (anonymous access: none)" ;;
|
||||
*) echo "s3-init: bucket is NOT private: $access"; exit 1 ;;
|
||||
esac
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
"""Очистка незавершённых загрузок и сверка PostgreSQL <-> S3.
|
||||
|
||||
Запускается отдельным сервисом `janitor` (раз в JANITOR_INTERVAL_SECONDS) или вручную:
|
||||
|
||||
python janitor.py --once [--dry-run] [--grace-seconds N]
|
||||
|
||||
Проход состоит из двух шагов. Оба трогают только то, что старше grace period
|
||||
(ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч, минимум 2 x UPLOAD_TIMEOUT_SECONDS): загрузка,
|
||||
которая ещё идёт, столько не живёт, поэтому свежие и готовые вложения не удаляются.
|
||||
|
||||
1. Брошенные намерения: строки state=uploading старше grace. Для каждой (под FOR UPDATE SKIP LOCKED)
|
||||
удалить объект в S3 (идемпотентно) и затем строку. Если S3 недоступен, строка остаётся
|
||||
до следующего прохода — указатель на объект не теряется.
|
||||
2. Сирота в S3: объект под originals/ старше grace, для которого в БД нет строки ни в каком
|
||||
состоянии. API кладёт объект только после COMMIT строки-намерения, поэтому у живой загрузки
|
||||
строка всегда есть. Удаляется.
|
||||
Заодно отчёт: ready-строки, чьего объекта нет в bucket (missing). Их не трогаем — это потеря
|
||||
данных, её разбирает человек (восстановление из резервной копии), а не автоматическая очистка.
|
||||
|
||||
Одновременно работает только один janitor: pg_try_advisory_lock.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import settings
|
||||
import storage
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
log = logging.getLogger("messenger.janitor")
|
||||
|
||||
JANITOR_LOCK = 20260404
|
||||
BATCH = 100
|
||||
STOP = threading.Event()
|
||||
|
||||
|
||||
def connect():
|
||||
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||||
|
||||
|
||||
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
|
||||
"""Шаг 1. Возвращает (найдено, удалено)."""
|
||||
found = deleted = 0
|
||||
while not STOP.is_set():
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""SELECT id, object_key FROM attachments
|
||||
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
|
||||
ORDER BY created_at LIMIT %s
|
||||
FOR UPDATE SKIP LOCKED""",
|
||||
(grace, BATCH),
|
||||
).fetchall()
|
||||
found += len(rows)
|
||||
if dry_run or not rows:
|
||||
return found, deleted
|
||||
for row in rows:
|
||||
storage.delete(row["object_key"]) # недоступен S3 -> исключение, транзакция откатится
|
||||
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
|
||||
deleted += 1
|
||||
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
|
||||
if len(rows) < BATCH:
|
||||
return found, deleted
|
||||
return found, deleted
|
||||
|
||||
|
||||
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
|
||||
"""Шаг 2. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
|
||||
started = datetime.now(timezone.utc)
|
||||
cutoff = started - timedelta(seconds=grace)
|
||||
scanned = orphans = orphans_deleted = 0
|
||||
listed = set()
|
||||
page: list[tuple[str, datetime]] = []
|
||||
|
||||
def check(page_items):
|
||||
nonlocal orphans, orphans_deleted
|
||||
keys = [key for key, _ in page_items]
|
||||
known = {r["object_key"] for r in conn.execute(
|
||||
"SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,)).fetchall()}
|
||||
for key, modified in page_items:
|
||||
if key in known or modified >= cutoff:
|
||||
continue
|
||||
orphans += 1
|
||||
if not dry_run:
|
||||
# Повторная проверка прямо перед удалением: строка могла появиться между запросами.
|
||||
if conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (key,)).fetchone() is None:
|
||||
storage.delete(key)
|
||||
orphans_deleted += 1
|
||||
log.info("deleted orphan object %s (no row in PostgreSQL)", key)
|
||||
|
||||
for key, modified, _size in storage.list_objects():
|
||||
scanned += 1
|
||||
listed.add(key)
|
||||
page.append((key, modified))
|
||||
if len(page) >= 1000:
|
||||
check(page)
|
||||
page = []
|
||||
if page:
|
||||
check(page)
|
||||
|
||||
missing = [str(r["id"]) for r in conn.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE state = 'ready' AND updated_at < %s", (started,)).fetchall()
|
||||
if r["object_key"] not in listed]
|
||||
if missing:
|
||||
log.error("ready attachments without object in S3 (not deleted, needs restore): %s", ", ".join(missing[:20]))
|
||||
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
|
||||
"missing_objects": len(missing)}
|
||||
|
||||
|
||||
def run_once(grace: int, dry_run: bool) -> dict | None:
|
||||
started = time.monotonic()
|
||||
with connect() as conn:
|
||||
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
|
||||
log.info("another janitor is running, skip")
|
||||
return None
|
||||
try:
|
||||
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
|
||||
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
|
||||
"stale_uploads_deleted": deleted, **sweep_objects(conn, grace, dry_run)}
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
|
||||
report["seconds"] = round(time.monotonic() - started, 3)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads and orphan S3 objects")
|
||||
parser.add_argument("--once", action="store_true", help="one pass and exit")
|
||||
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
|
||||
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
|
||||
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
|
||||
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
signal.signal(sig, lambda *_: STOP.set())
|
||||
while True:
|
||||
try:
|
||||
report = run_once(args.grace_seconds, args.dry_run)
|
||||
if report is not None:
|
||||
print(json.dumps(report), flush=True)
|
||||
failed = False
|
||||
except (psycopg.Error, storage.StorageUnavailable) as exc:
|
||||
log.warning("pass failed, will retry: %s", type(exc).__name__)
|
||||
failed = True
|
||||
if args.once:
|
||||
raise SystemExit(1 if failed else 0)
|
||||
if STOP.wait(settings.JANITOR_INTERVAL_SECONDS):
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,583 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Header, Query, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
import attachments
|
||||
import auth
|
||||
import db
|
||||
import settings
|
||||
import storage
|
||||
import uploads
|
||||
from errors import ApiError, bad_request, unauthorized, unavailable
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
log = logging.getLogger("messenger")
|
||||
|
||||
DRAINING = threading.Event()
|
||||
|
||||
|
||||
class Server(uvicorn.Server):
|
||||
"""Остановка по SIGTERM в два шага.
|
||||
|
||||
1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS
|
||||
HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы.
|
||||
2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы
|
||||
(не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM,
|
||||
получал бы 502 от балансировщика.
|
||||
Повторный сигнал останавливает сразу.
|
||||
"""
|
||||
|
||||
def handle_exit(self, sig, frame):
|
||||
if DRAINING.is_set():
|
||||
return super().handle_exit(sig, frame)
|
||||
DRAINING.set()
|
||||
log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s",
|
||||
sig, settings.SHUTDOWN_DRAIN_SECONDS)
|
||||
threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
settings.check()
|
||||
# Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему.
|
||||
await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS)
|
||||
# Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3.
|
||||
Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True)
|
||||
db.pool.open()
|
||||
log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s",
|
||||
settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, settings.SESSION_TTL_SECONDS,
|
||||
settings.S3_BUCKET)
|
||||
yield
|
||||
db.pool.close()
|
||||
auth.close()
|
||||
log.info("Stopped: connections closed, data stays in PostgreSQL and Redis")
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH)
|
||||
|
||||
@field_validator("display_name", "password")
|
||||
@classmethod
|
||||
def validate_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("value cannot be blank")
|
||||
return value
|
||||
|
||||
|
||||
class User(BaseModel):
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class Session(BaseModel):
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: datetime
|
||||
user: User
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def validate_title(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("title cannot be empty")
|
||||
return value
|
||||
|
||||
|
||||
class Chat(BaseModel):
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
items: list[Chat]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
text: str = Field(max_length=4000)
|
||||
# С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением.
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_content(self):
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("attachment_ids must be unique")
|
||||
if not self.attachment_ids and not self.text.strip():
|
||||
raise ValueError("text cannot be empty without attachments")
|
||||
return self
|
||||
|
||||
|
||||
class Message(BaseModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[UUID]
|
||||
|
||||
|
||||
class Variant(BaseModel):
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class Attachment(BaseModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: Literal["ready"]
|
||||
variants: list[Variant]
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
items: list[Message]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request,
|
||||
code: str,
|
||||
message: str,
|
||||
status_code: int,
|
||||
headers: dict | None = None,
|
||||
):
|
||||
request_id = request.state.request_id
|
||||
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
headers={"X-Request-Id": request_id, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
return error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(
|
||||
request: Request,
|
||||
exc: RequestValidationError,
|
||||
):
|
||||
return error_response(
|
||||
request,
|
||||
"invalid_request",
|
||||
"Invalid request data",
|
||||
400,
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
|
||||
if exc.status_code == 404:
|
||||
return error_response(request, "not_found", "Not found", 404)
|
||||
return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def general_exception_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
):
|
||||
return error_response(
|
||||
request,
|
||||
"internal_error",
|
||||
"Internal server error",
|
||||
500,
|
||||
)
|
||||
|
||||
PUBLIC_ENDPOINTS = {
|
||||
("POST", "/api/v1/users"),
|
||||
("POST", "/api/v1/auth/sessions"),
|
||||
}
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(request: Request, call_next):
|
||||
request_id = str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
started = time.perf_counter()
|
||||
path = request.url.path
|
||||
|
||||
try:
|
||||
if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS:
|
||||
request.state.session = await run_in_threadpool(
|
||||
auth.check_bearer, request.headers.get("Authorization")
|
||||
)
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
|
||||
except Exception:
|
||||
log.exception("Unhandled error, request_id=%s", request_id)
|
||||
response = error_response(request, "internal_error", "Internal server error", 500)
|
||||
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила
|
||||
# Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки.
|
||||
level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO
|
||||
log.log(
|
||||
level,
|
||||
"%s %s -> %s (%.1f ms) request_id=%s",
|
||||
request.method, path, response.status_code,
|
||||
(time.perf_counter() - started) * 1000, request_id,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def current_user_id(request: Request) -> UUID:
|
||||
session = getattr(request.state, "session", None)
|
||||
if session is None:
|
||||
raise unauthorized("Bearer token is required")
|
||||
return session.user_id
|
||||
|
||||
|
||||
def encode_cursor(
|
||||
created_at: datetime,
|
||||
object_id: UUID,
|
||||
*,
|
||||
chat_id: UUID | None = None,
|
||||
user_id: UUID | None = None,
|
||||
) -> str:
|
||||
data = {
|
||||
"created_at": created_at.isoformat(),
|
||||
"id": str(object_id),
|
||||
}
|
||||
|
||||
if chat_id is not None:
|
||||
data["chat_id"] = str(chat_id)
|
||||
|
||||
if user_id is not None:
|
||||
data["user_id"] = str(user_id)
|
||||
|
||||
raw = json.dumps(data).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str):
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
|
||||
created_at = datetime.fromisoformat(data["created_at"])
|
||||
|
||||
if created_at.tzinfo is None:
|
||||
raise ValueError("Invalid cursor")
|
||||
|
||||
if not isinstance(data["id"], str):
|
||||
raise ValueError("Invalid cursor")
|
||||
|
||||
object_id = UUID(data["id"])
|
||||
|
||||
chat_id = UUID(data["chat_id"]) if "chat_id" in data else None
|
||||
user_id = UUID(data["user_id"]) if "user_id" in data else None
|
||||
|
||||
return created_at, object_id, chat_id, user_id
|
||||
|
||||
except (
|
||||
ValueError,
|
||||
KeyError,
|
||||
TypeError,
|
||||
AttributeError,
|
||||
json.JSONDecodeError,
|
||||
binascii.Error,
|
||||
):
|
||||
raise ValueError("Invalid cursor")
|
||||
|
||||
|
||||
def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None):
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor)
|
||||
except ValueError:
|
||||
raise bad_request("Invalid cursor") from None
|
||||
if cursor_chat_id != chat_id or cursor_user_id != user_id:
|
||||
raise bad_request("Invalid cursor")
|
||||
return created_at, object_id
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
if DRAINING.is_set():
|
||||
raise unavailable("Instance is shutting down")
|
||||
db.ping()
|
||||
auth.ping_redis()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", response_model=User, status_code=201)
|
||||
def create_user(user_data: CreateUser):
|
||||
password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется
|
||||
return db.create_user(user_data.username, user_data.display_name, password_hash)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201)
|
||||
def create_session(
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
):
|
||||
credentials = auth.parse_basic(authorization)
|
||||
if credentials is None:
|
||||
raise auth.login_failed()
|
||||
username, password = credentials
|
||||
|
||||
found = None
|
||||
if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH:
|
||||
found = db.get_user_with_password(username)
|
||||
if not auth.password_matches(found["password_hash"] if found else None, password):
|
||||
raise auth.login_failed()
|
||||
|
||||
token, expires_at = auth.create_session(found["id"])
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
user = {key: found[key] for key in ("id", "username", "display_name", "created_at")}
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def delete_session(request: Request):
|
||||
current_user_id(request)
|
||||
auth.delete_session(request.state.session)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=User)
|
||||
def get_current_user(request: Request):
|
||||
user = db.get_user(current_user_id(request))
|
||||
if user is None:
|
||||
raise unauthorized("Session is invalid or expired")
|
||||
return user
|
||||
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", response_model=Chat, status_code=201)
|
||||
def create_chat(request: Request, chat_data: CreateChat):
|
||||
user_id = current_user_id(request)
|
||||
|
||||
if user_id not in chat_data.member_ids:
|
||||
raise bad_request("Creator must be a chat member")
|
||||
|
||||
if len(chat_data.member_ids) != len(set(chat_data.member_ids)):
|
||||
raise bad_request("Duplicate member IDs")
|
||||
return db.create_chat(chat_data.title, chat_data.member_ids)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def list_chats(
|
||||
request: Request,
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None),
|
||||
):
|
||||
user_id = current_user_id(request)
|
||||
after = page_after(cursor, user_id=user_id)
|
||||
|
||||
rows = db.list_chats(user_id, after, limit)
|
||||
items = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = items[-1]
|
||||
next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id)
|
||||
|
||||
return ChatPage(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
|
||||
def get_chat(request: Request, chat_id: UUID):
|
||||
return db.get_chat(chat_id, current_user_id(request))
|
||||
|
||||
|
||||
def request_fingerprint(data: BaseModel) -> bytes:
|
||||
"""SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют.
|
||||
|
||||
Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []}
|
||||
равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними.
|
||||
"""
|
||||
body = data.model_dump(mode="json")
|
||||
if not body.get("attachment_ids"):
|
||||
body.pop("attachment_ids", None)
|
||||
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
response_model=Message,
|
||||
status_code=201,
|
||||
)
|
||||
def create_message(
|
||||
request: Request,
|
||||
message_data: CreateMessage,
|
||||
chat_id: UUID,
|
||||
idempotency_key: str | None = Header(default=None),
|
||||
):
|
||||
# С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику)
|
||||
# вернёт то же сообщение, а не создаст второе. См. db.add_message.
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -")
|
||||
return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids,
|
||||
idempotency_key, request_fingerprint(message_data))
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
response_model=MessagePage,
|
||||
)
|
||||
def list_messages(
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None),
|
||||
):
|
||||
user_id = current_user_id(request)
|
||||
after = page_after(cursor, chat_id=chat_id)
|
||||
|
||||
rows = db.list_messages(chat_id, user_id, after, limit)
|
||||
items = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = items[-1]
|
||||
next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id)
|
||||
|
||||
return MessagePage(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
response_model=Attachment,
|
||||
status_code=201,
|
||||
)
|
||||
async def upload_attachment(
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
filename: str | None = Query(default=None),
|
||||
):
|
||||
"""Синхронная загрузка PNG/JPEG сырыми байтами. 201 — только когда байты в S3 и строка ready в БД.
|
||||
|
||||
Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415)
|
||||
-> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413),
|
||||
проверка содержимого (415/413) и запись (attachments.store).
|
||||
"""
|
||||
user_id = current_user_id(request)
|
||||
try:
|
||||
await run_in_threadpool(db.check_member, chat_id, user_id)
|
||||
name = uploads.validate_filename(filename)
|
||||
declared = uploads.declared_type(request.headers.get("content-type"))
|
||||
uploads.declared_length(request.headers.get("content-length"))
|
||||
except ApiError:
|
||||
await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке
|
||||
raise
|
||||
upload = await uploads.receive(request)
|
||||
try:
|
||||
image = await run_in_threadpool(uploads.inspect, upload, declared)
|
||||
row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image)
|
||||
finally:
|
||||
upload.file.close() # временный файл удаляется сразу; оригинал остаётся только в S3
|
||||
return attachments.public(row)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
|
||||
def get_attachment(request: Request, attachment_id: UUID):
|
||||
return attachments.public(db.get_attachment(attachment_id, current_user_id(request)))
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(
|
||||
request: Request,
|
||||
attachment_id: UUID,
|
||||
variant: Literal["original"] = Query(default="original"),
|
||||
):
|
||||
"""Байты оригинала через API (без redirect): сначала проверка участника чата, потом S3."""
|
||||
row = db.get_attachment(attachment_id, current_user_id(request))
|
||||
try:
|
||||
stream = storage.open_read(row["object_key"])
|
||||
except storage.ObjectMissing:
|
||||
log.error("Object of ready attachment %s is missing in S3", row["id"])
|
||||
raise unavailable("File is temporarily unavailable") from None
|
||||
except storage.StorageUnavailable:
|
||||
raise unavailable("File storage is temporarily unavailable") from None
|
||||
headers = {
|
||||
"Content-Length": str(row["size_bytes"]),
|
||||
"Content-Disposition": "inline; filename*=UTF-8''" + quote(row["filename"], safe=""),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Cache-Control": "private, no-store",
|
||||
"ETag": '"' + row["sha256"] + '"',
|
||||
}
|
||||
return StreamingResponse(storage.iter_chunks(stream), media_type=row["content_type"], headers=headers)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Server(uvicorn.Config(
|
||||
app,
|
||||
host=settings.APP_HOST,
|
||||
port=settings.APP_PORT,
|
||||
timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS,
|
||||
access_log=False,
|
||||
)).run()
|
||||
@@ -0,0 +1,8 @@
|
||||
import logging
|
||||
|
||||
import db
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
|
||||
if __name__ == "__main__":
|
||||
db.wait_and_migrate()
|
||||
@@ -0,0 +1,33 @@
|
||||
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
|
||||
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
|
||||
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8');
|
||||
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8');
|
||||
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8');
|
||||
|
||||
ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400);
|
||||
ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400);
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000);
|
||||
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE messages ADD COLUMN idempotency_key text
|
||||
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$');
|
||||
|
||||
ALTER TABLE messages ADD COLUMN request_hash bytea
|
||||
CHECK (octet_length(request_hash) = 32);
|
||||
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together
|
||||
CHECK ((idempotency_key IS NULL) = (request_hash IS NULL));
|
||||
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key
|
||||
UNIQUE (chat_id, sender_id, idempotency_key);
|
||||
@@ -0,0 +1,48 @@
|
||||
-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением.
|
||||
-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать.
|
||||
|
||||
-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата.
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
owner_id uuid NOT NULL,
|
||||
filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width integer NOT NULL CHECK (width >= 1),
|
||||
height integer NOT NULL CHECK (height >= 1),
|
||||
-- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит.
|
||||
object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'),
|
||||
-- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна;
|
||||
-- ready: объект в S3 сохранён, можно читать и прикреплять.
|
||||
state text NOT NULL CHECK (state IN ('uploading', 'ready')),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CHECK (width::bigint * height <= 40000000),
|
||||
UNIQUE (id, chat_id),
|
||||
-- владелец на момент загрузки был участником именно этого чата
|
||||
FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
|
||||
-- janitor ищет старые незавершённые загрузки
|
||||
CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading';
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, position),
|
||||
UNIQUE (message_id, attachment_id),
|
||||
-- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id),
|
||||
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
|
||||
|
||||
-- С lab4 текст может быть пустым, если есть вложение (проверяет API).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000);
|
||||
@@ -0,0 +1,29 @@
|
||||
fastapi==0.141.1
|
||||
uvicorn==0.54.0
|
||||
pydantic==2.13.5
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
argon2-cffi==25.1.0
|
||||
redis==6.4.0
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi-bindings==26.1.0
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
pycparser==3.1
|
||||
pydantic_core==2.46.5
|
||||
starlette==1.7.0
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
boto3==1.43.116
|
||||
botocore==1.43.116
|
||||
s3transfer==0.19.3
|
||||
jmespath==1.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
six==1.17.0
|
||||
urllib3==2.8.0
|
||||
pillow==12.3.0
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init.
|
||||
|
||||
Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает.
|
||||
Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py):
|
||||
admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init;
|
||||
messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor.
|
||||
Значения ключей не печатаются.
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
import settings
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
|
||||
|
||||
def client(key_id: str, secret: str):
|
||||
return boto3.client(
|
||||
"s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION,
|
||||
aws_access_key_id=key_id, aws_secret_access_key=secret,
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2,
|
||||
read_timeout=10, retries={"total_max_attempts": 1},
|
||||
request_checksum_calculation="when_required", response_checksum_validation="when_required"),
|
||||
)
|
||||
|
||||
|
||||
def code(exc: ClientError) -> str:
|
||||
return exc.response.get("Error", {}).get("Code", "?")
|
||||
|
||||
|
||||
def anonymous_status(url: str, method: str = "GET") -> int:
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply:
|
||||
return reply.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
bucket = settings.S3_BUCKET
|
||||
admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"),
|
||||
settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY"))
|
||||
api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY)
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
admin.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
if time.monotonic() > deadline:
|
||||
print(f"s3-init: S3 is not ready: {type(exc).__name__}")
|
||||
return 1
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
admin.create_bucket(Bucket=bucket)
|
||||
print(f"s3-init: bucket {bucket} created")
|
||||
except ClientError as exc:
|
||||
if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
print(f"s3-init: cannot create bucket: {code(exc)}")
|
||||
return 1
|
||||
print(f"s3-init: bucket {bucket} exists")
|
||||
|
||||
# Ключ API работает со своим bucket ...
|
||||
probe = "originals/s3-init-probe-" + uuid.uuid4().hex
|
||||
api.put_object(Bucket=bucket, Key=probe, Body=b"probe")
|
||||
api.get_object(Bucket=bucket, Key=probe)["Body"].read()
|
||||
api.list_objects_v2(Bucket=bucket, MaxKeys=1)
|
||||
api.delete_object(Bucket=bucket, Key=probe)
|
||||
print("s3-init: api key can put/get/list/delete objects of its bucket")
|
||||
# ... и не может администрировать хранилище.
|
||||
intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8]
|
||||
try:
|
||||
api.create_bucket(Bucket=intruder)
|
||||
admin.delete_bucket(Bucket=intruder)
|
||||
print("s3-init: api key must NOT be able to create buckets")
|
||||
return 1
|
||||
except ClientError as exc:
|
||||
print(f"s3-init: api key cannot create buckets ({code(exc)})")
|
||||
|
||||
# Приватность: анонимные запросы без подписи.
|
||||
base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket
|
||||
statuses = {"list": anonymous_status(base + "?list-type=2"),
|
||||
"get": anonymous_status(base + "/originals/any"),
|
||||
"put": anonymous_status(base + "/originals/any", "PUT")}
|
||||
if any(status not in (401, 403) for status in statuses.values()):
|
||||
print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}")
|
||||
return 1
|
||||
print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,77 @@
|
||||
import json
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
|
||||
|
||||
def write(name: str, value: str) -> None:
|
||||
with open(SECRETS / name, "w", encoding="utf-8", newline="") as file:
|
||||
file.write(value)
|
||||
(SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore
|
||||
|
||||
|
||||
def bucket_name() -> str:
|
||||
"""S3_BUCKET из .env (если задан), иначе значение по умолчанию."""
|
||||
env = ROOT / ".env"
|
||||
if env.exists():
|
||||
for line in env.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip().startswith("S3_BUCKET="):
|
||||
return line.split("=", 1)[1].strip()
|
||||
return "messenger-attachments"
|
||||
|
||||
|
||||
def s3_config(bucket: str) -> str:
|
||||
"""Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket.
|
||||
|
||||
Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects)
|
||||
с суффиксом :<bucket> действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет.
|
||||
Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются.
|
||||
"""
|
||||
def read(name):
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
config = {"identities": [
|
||||
{"name": "admin",
|
||||
"credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "messenger-api",
|
||||
"credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
|
||||
]}
|
||||
return json.dumps(config, indent=2)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes)."""
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
created = []
|
||||
if not (SECRETS / "postgres_password").exists():
|
||||
db_password = secrets.token_hex(24)
|
||||
redis_password = secrets.token_hex(24)
|
||||
write("postgres_password", db_password)
|
||||
write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger")
|
||||
write("cache_url", f"redis://:{redis_password}@redis:6379/0")
|
||||
write("redis_auth", f"requirepass {redis_password}\n")
|
||||
created += ["postgres_password", "database_url", "cache_url", "redis_auth"]
|
||||
if not (SECRETS / "s3_root_password").exists():
|
||||
# admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init
|
||||
write("s3_root_user", "root-" + secrets.token_hex(6))
|
||||
write("s3_root_password", secrets.token_hex(24))
|
||||
created += ["s3_root_user", "s3_root_password"]
|
||||
if not (SECRETS / "s3_secret_access_key").exists():
|
||||
# отдельный ключ API с политикой только на свой bucket (создаёт s3-init)
|
||||
write("s3_access_key_id", "api-" + secrets.token_hex(8))
|
||||
write("s3_secret_access_key", secrets.token_hex(24))
|
||||
created += ["s3_access_key_id", "s3_secret_access_key"]
|
||||
if not (SECRETS / "s3_config.json").exists():
|
||||
write("s3_config.json", s3_config(bucket_name()))
|
||||
created.append("s3_config.json")
|
||||
if created:
|
||||
print("Created in secrets/: " + ", ".join(created) + " (values are not printed).")
|
||||
else:
|
||||
print("secrets/ is complete, nothing changed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
@echo off
|
||||
rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then api and janitor, then the balancer. Volumes are kept.
|
||||
rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
|
||||
rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart postgres redis s3 || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 || exit /b 1
|
||||
docker compose restart api janitor || exit /b 1
|
||||
docker compose restart lb || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps api lb || exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
# Restart hook for tests/resilience.py --mode persistence (Linux/macOS).
|
||||
# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then all api replicas and janitor, then the balancer.
|
||||
# Volumes are kept (no down -v), nothing is re-seeded.
|
||||
# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
|
||||
# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart postgres redis s3
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3
|
||||
docker compose restart api janitor
|
||||
docker compose restart lb
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps api lb
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py.
|
||||
python "%~dp0stop_one.py"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID.
|
||||
# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd.
|
||||
set -eu
|
||||
exec python3 "$(dirname "$0")/stop_one.py"
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STOP_TIMEOUT_SECONDS = 8
|
||||
|
||||
|
||||
def main() -> int:
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{12,64}", target):
|
||||
print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr)
|
||||
return 2
|
||||
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
|
||||
capture_output=True, text=True, check=True).stdout.split()
|
||||
matches = [container for container in running if container.startswith(target)]
|
||||
if len(matches) != 1:
|
||||
print(f"no running api container with id {target}", file=sys.stderr)
|
||||
return 1
|
||||
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
|
||||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
print(f"stopped api container {matches[0][:12]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
import os
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def env_str(name: str, default: str) -> str:
|
||||
return os.environ.get(name, default).strip()
|
||||
|
||||
|
||||
def env_int(name: str, default: int) -> int:
|
||||
value = int(env_str(name, str(default)))
|
||||
if value < 1:
|
||||
raise ValueError(f"{name} must be >= 1")
|
||||
return value
|
||||
|
||||
|
||||
def read_secret(file_variable: str, plain_variable: str) -> str:
|
||||
path = env_str(file_variable, "")
|
||||
if path:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return env_str(plain_variable, "")
|
||||
|
||||
|
||||
APP_HOST = env_str("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = env_int("APP_PORT", 8080)
|
||||
APP_ENV = env_str("APP_ENV", "lab")
|
||||
APP_VERSION = env_str("APP_VERSION", "dev")
|
||||
|
||||
INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname()
|
||||
|
||||
SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_STORE = env_str("SESSION_STORE", "redis")
|
||||
|
||||
DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
|
||||
DB_POOL_MIN = env_int("DB_POOL_MIN", 1)
|
||||
DB_POOL_MAX = env_int("DB_POOL_MAX", 10)
|
||||
DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3)
|
||||
DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3)
|
||||
DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
|
||||
REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL")
|
||||
REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20)
|
||||
REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500)
|
||||
|
||||
MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations")))
|
||||
SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60)
|
||||
|
||||
# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket.
|
||||
S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333")
|
||||
S3_REGION = env_str("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY")
|
||||
S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1)
|
||||
S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5)
|
||||
S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2)
|
||||
S3_POOL_MAX = env_int("S3_POOL_MAX", 20)
|
||||
|
||||
# Вложения: лимиты из контракта и защита памяти/времени.
|
||||
MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)
|
||||
MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000)
|
||||
UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60)
|
||||
UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10)
|
||||
DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2)
|
||||
UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads")
|
||||
|
||||
# Очистка незавершённых загрузок (janitor.py).
|
||||
ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600)
|
||||
JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300)
|
||||
|
||||
SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3)
|
||||
SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10)
|
||||
|
||||
|
||||
def check() -> None:
|
||||
"""Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе."""
|
||||
if not DATABASE_URL:
|
||||
raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE")
|
||||
if not REDIS_URL:
|
||||
raise RuntimeError("Redis address is not set: CACHE_URL_FILE")
|
||||
if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY:
|
||||
raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE")
|
||||
if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS:
|
||||
raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS")
|
||||
if SESSION_STORE != "redis":
|
||||
raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution")
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
"""Приватное S3-хранилище (SeaweedFS) для байтов вложений.
|
||||
|
||||
API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket
|
||||
(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер,
|
||||
имя файла пользователя в ключ не попадает.
|
||||
"""
|
||||
import base64
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
import settings
|
||||
|
||||
log = logging.getLogger("messenger.storage")
|
||||
|
||||
ORIGINALS_PREFIX = "originals/"
|
||||
READ_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class StorageUnavailable(Exception):
|
||||
"""S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""Объекта нет в bucket (NoSuchKey / 404)."""
|
||||
|
||||
|
||||
def _make_client(connect_timeout: float, read_timeout: float, attempts: int):
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.S3_ENDPOINT,
|
||||
region_name=settings.S3_REGION,
|
||||
aws_access_key_id=settings.S3_ACCESS_KEY_ID or None,
|
||||
aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"}, # http://s3:8333/<bucket>/<key>
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую
|
||||
max_pool_connections=settings.S3_POOL_MAX,
|
||||
# Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно.
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS)
|
||||
# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor.
|
||||
_quick_client = _make_client(1, 2, 1)
|
||||
|
||||
|
||||
def _is_missing(exc: ClientError) -> bool:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
return code in ("NoSuchKey", "404", "NotFound") or status == 404
|
||||
|
||||
|
||||
def _unavailable(action: str, exc: Exception) -> StorageUnavailable:
|
||||
detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else ""
|
||||
log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail)
|
||||
return StorageUnavailable()
|
||||
|
||||
|
||||
def new_key(object_id: str) -> str:
|
||||
return ORIGINALS_PREFIX + object_id
|
||||
|
||||
|
||||
def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
|
||||
"""Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект."""
|
||||
body.seek(0)
|
||||
try:
|
||||
_client.put_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=key,
|
||||
Body=body,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
|
||||
)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("put", exc) from None
|
||||
|
||||
|
||||
def open_read(key: str):
|
||||
"""GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после."""
|
||||
try:
|
||||
return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"]
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
raise ObjectMissing(key) from None
|
||||
raise _unavailable("get", exc) from None
|
||||
except BotoCoreError as exc:
|
||||
raise _unavailable("get", exc) from None
|
||||
|
||||
|
||||
def iter_chunks(stream) -> Iterator[bytes]:
|
||||
"""Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание."""
|
||||
try:
|
||||
while True:
|
||||
chunk = stream.read(READ_CHUNK)
|
||||
if not chunk:
|
||||
return
|
||||
yield chunk
|
||||
except (BotoCoreError, OSError) as exc:
|
||||
# Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length).
|
||||
log.warning("S3 stream interrupted: %s", type(exc).__name__)
|
||||
raise
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
|
||||
def delete(key: str, quick: bool = False) -> None:
|
||||
"""Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа."""
|
||||
try:
|
||||
(_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("delete", exc) from None
|
||||
|
||||
|
||||
def exists(key: str) -> bool:
|
||||
try:
|
||||
_client.head_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
return True
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
return False
|
||||
raise _unavailable("head", exc) from None
|
||||
except BotoCoreError as exc:
|
||||
raise _unavailable("head", exc) from None
|
||||
|
||||
|
||||
def list_objects(prefix: str = ORIGINALS_PREFIX) -> Iterator[tuple[str, datetime, int]]:
|
||||
"""Все объекты под prefix: (key, last_modified, size). Постранично, по 1000."""
|
||||
paginator = _client.get_paginator("list_objects_v2")
|
||||
try:
|
||||
for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix):
|
||||
for item in page.get("Contents", []):
|
||||
yield item["Key"], item["LastModified"], item["Size"]
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("list", exc) from None
|
||||
|
||||
|
||||
def ping() -> None:
|
||||
try:
|
||||
_client.head_bucket(Bucket=settings.S3_BUCKET)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("head_bucket", exc) from None
|
||||
@@ -0,0 +1,78 @@
|
||||
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
|
||||
from common import call, check, compose_output, docker, new_user, seen_instances
|
||||
|
||||
|
||||
def main():
|
||||
info = json.loads(docker("info", "--format", "{{json .}}"))
|
||||
print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
|
||||
f"client Python {platform.python_version()}")
|
||||
print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, "
|
||||
f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, "
|
||||
f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}")
|
||||
|
||||
print()
|
||||
print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip())
|
||||
for one_shot in ("migrate", "s3-init"):
|
||||
status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip()
|
||||
check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0")
|
||||
|
||||
config = json.loads(compose_output("config", "--format", "json"))
|
||||
services = config["services"]
|
||||
containers = compose_output("ps", "-q").split()
|
||||
rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()]
|
||||
print()
|
||||
print("container image cpus memory host ports networks")
|
||||
api_count = 0
|
||||
for row in sorted(rows, key=lambda r: r["Name"]):
|
||||
labels = row["Config"]["Labels"]
|
||||
service = labels.get("com.docker.compose.service")
|
||||
limits = row["HostConfig"]
|
||||
cpus = limits.get("NanoCpus", 0) / 1e9
|
||||
memory = limits.get("Memory", 0) // 2**20
|
||||
ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values()
|
||||
for b in (bindings or [])})
|
||||
networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"])
|
||||
print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB "
|
||||
f"{','.join(ports) or '-':<23} {','.join(networks)}")
|
||||
check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit")
|
||||
if service == "api":
|
||||
api_count += 1
|
||||
check(not ports, "api must not publish ports: everything goes through the balancer")
|
||||
if service == "lb":
|
||||
check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only")
|
||||
if service in ("postgres", "redis", "s3", "janitor"):
|
||||
check(not ports and networks == ["data"], f"{service} must be only in the internal data network")
|
||||
check(api_count >= 2, "need at least 2 api replicas")
|
||||
check(all("container_name" not in s for s in services.values()), "container_name prevents scaling")
|
||||
check(config["networks"]["data"].get("internal") is True, "data network must be internal")
|
||||
check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis")
|
||||
|
||||
print()
|
||||
for name, service in sorted(services.items()):
|
||||
secrets = sorted(s["source"] for s in service.get("secrets", []))
|
||||
print(f"secrets of {name}: {secrets or '-'}")
|
||||
check(not services["lb"].get("secrets"), "lb gets no secrets")
|
||||
check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url")
|
||||
for name in ("api", "janitor"):
|
||||
names = {s["source"] for s in services[name]["secrets"]}
|
||||
check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys")
|
||||
check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key")
|
||||
|
||||
user = new_user("stack")
|
||||
served = seen_instances(user["token"])
|
||||
print()
|
||||
print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}")
|
||||
check(len(served) >= 2, "requests are not balanced")
|
||||
health = call("GET", "/health/ready", 200)
|
||||
print(f"/health/ready -> {health.status} from {health.instance}")
|
||||
print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every "
|
||||
f"container, postgres/redis/s3/janitor only in the internal data network, "
|
||||
f"S3 admin keys only in s3 and s3-init")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,293 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
PY = sys.executable
|
||||
|
||||
STEPS = [
|
||||
("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]),
|
||||
("stack", "stand layout: replicas, ports, limits, networks, secrets, S3", [PY, "student-tests/check_stack.py"]),
|
||||
("smoke", "public smoke via the balancer (make test), with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]),
|
||||
("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids",
|
||||
[PY, "student-tests/test_attachments.py"]),
|
||||
("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap",
|
||||
[PY, "student-tests/test_s3_security.py"]),
|
||||
("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage",
|
||||
[PY, "student-tests/test_consistency.py"]),
|
||||
("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas",
|
||||
[PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]),
|
||||
("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]),
|
||||
("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]),
|
||||
("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]),
|
||||
("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]),
|
||||
("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]),
|
||||
("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]),
|
||||
("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]),
|
||||
("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)",
|
||||
[PY, "student-tests/run_failover.py"]),
|
||||
("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL,
|
||||
"--duration", "30", "--concurrency", "4",
|
||||
"--output", "evidence/load.json"]),
|
||||
("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]),
|
||||
("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)",
|
||||
[PY, "student-tests/run_persistence.py"]),
|
||||
]
|
||||
|
||||
|
||||
def git(*args):
|
||||
try:
|
||||
return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return ""
|
||||
|
||||
|
||||
def commit():
|
||||
head = git("rev-parse", "--short", "HEAD") or "unknown"
|
||||
changed = [line for line in git("status", "--porcelain").splitlines()
|
||||
if not line[3:].startswith(("evidence/", "REPORT.md"))]
|
||||
return head + (" + uncommitted changes" if changed else "")
|
||||
|
||||
|
||||
def run_step(name, description, command):
|
||||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL)
|
||||
print(f"=== {name}: {description}", flush=True)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
seconds = time.monotonic() - started
|
||||
output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n")
|
||||
shown = " ".join(Path(part).name if part == PY else part for part in command)
|
||||
header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n"
|
||||
f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n")
|
||||
(EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8")
|
||||
tail = [line for line in output.splitlines() if line.strip()][-3:]
|
||||
for line in tail:
|
||||
print(" " + line)
|
||||
print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True)
|
||||
return result.returncode == 0, seconds
|
||||
|
||||
|
||||
def load_json(name):
|
||||
path = EVIDENCE / name
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None
|
||||
|
||||
|
||||
def measurements():
|
||||
"""Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md)."""
|
||||
lines = []
|
||||
try:
|
||||
info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True,
|
||||
text=True, check=True).stdout)
|
||||
docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM"
|
||||
except (OSError, ValueError, subprocess.CalledProcessError):
|
||||
docker_vm = "unknown"
|
||||
lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
|
||||
f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона "
|
||||
f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB "
|
||||
f"каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, "
|
||||
f"janitor 0,25 CPU / 192 MiB.")
|
||||
lines.append("")
|
||||
|
||||
failover = EVIDENCE / "failover.txt"
|
||||
results = []
|
||||
if failover.exists():
|
||||
text = failover.read_text(encoding="utf-8")
|
||||
stopped = re.findall(r"the hook stopped (\w+)", text)
|
||||
results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)]
|
||||
if results:
|
||||
lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):")
|
||||
lines.append("")
|
||||
lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток "
|
||||
"(повторены с тем же ключом) | Первая запись после hook, с |")
|
||||
lines.append("| --- | --- | --- | --- | --- | --- |")
|
||||
for number, (result, container) in enumerate(zip(results, stopped), start=1):
|
||||
lines.append(f"| {number} | {container} | {result['observed_instances_before']} | "
|
||||
f"{result['successful_writes']} | {result['failed_attempts']} | "
|
||||
f"{result['first_recovery_seconds']} |")
|
||||
lines.append("")
|
||||
|
||||
memory = load_json("upload-memory.json")
|
||||
if memory:
|
||||
lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики "
|
||||
f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). "
|
||||
f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:")
|
||||
lines.append("")
|
||||
lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | "
|
||||
"Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |")
|
||||
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |")
|
||||
for run in memory["runs"]:
|
||||
for n, r in enumerate(run["replicas"]):
|
||||
head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | "
|
||||
f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |"
|
||||
cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-"
|
||||
lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | "
|
||||
f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |")
|
||||
naive = memory["single_file_extra_rss_mib"]
|
||||
lines.append("")
|
||||
lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в "
|
||||
f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка "
|
||||
f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в "
|
||||
f"{memory['api_limit_mib']} MiB.")
|
||||
lines.append("")
|
||||
|
||||
metrics = load_json("failover-metrics.json")
|
||||
if metrics:
|
||||
lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи "
|
||||
"без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):")
|
||||
lines.append("")
|
||||
lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок "
|
||||
"(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | "
|
||||
"Вернулась в балансировку после docker start, с |")
|
||||
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |")
|
||||
for r in metrics["runs"]:
|
||||
how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)"
|
||||
codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-"
|
||||
lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | "
|
||||
f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | "
|
||||
f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |")
|
||||
lines.append("")
|
||||
lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика "
|
||||
"после остановки не ответила ни разу.")
|
||||
lines.append("")
|
||||
|
||||
rolling = load_json("rolling-update.json")
|
||||
if rolling:
|
||||
during = rolling["during"]
|
||||
codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-"
|
||||
lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> "
|
||||
f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за "
|
||||
f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, "
|
||||
f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); "
|
||||
f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено "
|
||||
f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.")
|
||||
lines.append("")
|
||||
|
||||
load = load_json("load.json")
|
||||
if load:
|
||||
latency = load["successful_latency_ms"]
|
||||
lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, "
|
||||
f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных "
|
||||
f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных "
|
||||
f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.")
|
||||
lines.append("")
|
||||
|
||||
persistence = EVIDENCE / "persistence.txt"
|
||||
restart_seconds = None
|
||||
if persistence.exists():
|
||||
found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8"))
|
||||
restart_seconds = found.group(1) if found else None
|
||||
|
||||
impacts = []
|
||||
if metrics:
|
||||
for mode, what, expected in (
|
||||
("stop", "`docker stop` реплики (SIGTERM) под нагрузкой",
|
||||
"ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"),
|
||||
("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой",
|
||||
"единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")):
|
||||
runs = [r for r in metrics["runs"] if r["mode"] == mode]
|
||||
if runs:
|
||||
impacts.append(
|
||||
f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); "
|
||||
f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | "
|
||||
f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; "
|
||||
f"потерь 0, дублей 0; реплика вернулась через "
|
||||
f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |")
|
||||
if failover.exists() and results:
|
||||
impacts.append(
|
||||
f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, "
|
||||
f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: "
|
||||
f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через "
|
||||
f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |")
|
||||
if rolling:
|
||||
impacts.append(
|
||||
f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | "
|
||||
f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |")
|
||||
if restart_seconds:
|
||||
impacts.append(
|
||||
f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и "
|
||||
f"прежний токен живы | "
|
||||
f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |")
|
||||
consistency = EVIDENCE / "consistency.txt"
|
||||
if consistency.exists():
|
||||
text = consistency.read_text(encoding="utf-8")
|
||||
timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text)
|
||||
if len(timings) >= 2:
|
||||
impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, "
|
||||
f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | "
|
||||
f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |")
|
||||
impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за "
|
||||
f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |")
|
||||
if "PASS: S3 written but DB not committed" in text:
|
||||
impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | "
|
||||
"вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | "
|
||||
"GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | "
|
||||
"мусора не осталось; повторный проход ничего не делает |")
|
||||
if "PASS: an interrupted upload leaves no row" in text:
|
||||
impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | "
|
||||
"строк и объектов не прибавилось, временных файлов 0 | мгновенно |")
|
||||
if impacts:
|
||||
lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |")
|
||||
lines.append("| --- | --- | --- | --- |")
|
||||
lines.extend(impacts)
|
||||
lines.append("")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def update_report(block):
|
||||
report = ROOT / "REPORT.md"
|
||||
text = report.read_text(encoding="utf-8")
|
||||
begin, end = "<!-- measurements:begin -->", "<!-- measurements:end -->"
|
||||
if begin not in text or end not in text:
|
||||
print("REPORT.md has no measurement markers, only evidence/summary.md was written")
|
||||
return
|
||||
head, rest = text.split(begin, 1)
|
||||
_, tail = rest.split(end, 1)
|
||||
report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8")
|
||||
print("REPORT.md: measurement tables updated")
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
names = [name for name, _, _ in STEPS]
|
||||
if args[:1] == ["--from"]:
|
||||
if len(args) < 2 or args[1] not in names:
|
||||
raise SystemExit("usage: --from <step>; steps: " + ", ".join(names))
|
||||
selected = names[names.index(args[1]):]
|
||||
else:
|
||||
unknown = [a for a in args if a not in names]
|
||||
if unknown:
|
||||
raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names))
|
||||
selected = args or names
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
results = []
|
||||
for name, description, command in STEPS:
|
||||
if name not in selected:
|
||||
continue
|
||||
ok, seconds = run_step(name, description, command)
|
||||
results.append((name, description, ok, seconds))
|
||||
if not ok:
|
||||
print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.")
|
||||
print(f"After fixing: python student-tests/collect_evidence.py --from {name}")
|
||||
break
|
||||
block = measurements()
|
||||
summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |",
|
||||
"| --- | --- | --- | --- | --- |"]
|
||||
summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results]
|
||||
(EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8")
|
||||
print("evidence/summary.md written")
|
||||
passed = all(ok for _, _, ok, _ in results)
|
||||
if passed:
|
||||
update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков
|
||||
raise SystemExit(0 if passed else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,324 @@
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv")
|
||||
PASSWORD = "Test-password-123"
|
||||
|
||||
|
||||
class Response:
|
||||
def __init__(self, status, headers, data):
|
||||
self.status = status
|
||||
self.headers = {name.lower(): value for name, value in headers.items()}
|
||||
self.data = data
|
||||
try:
|
||||
self.json = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
self.json = None
|
||||
|
||||
@property
|
||||
def instance(self):
|
||||
"""X-Instance-Id: какая реплика API ответила (ID её контейнера)."""
|
||||
return self.headers.get("x-instance-id")
|
||||
|
||||
|
||||
def check(condition, message):
|
||||
if not condition:
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30):
|
||||
"""HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError.
|
||||
|
||||
Заодно проверяет то, что контракт требует от любого ответа:
|
||||
X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401.
|
||||
"""
|
||||
all_headers = dict(headers or {})
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
if data is not None:
|
||||
all_headers.setdefault("Content-Type", "application/json")
|
||||
if token is not None:
|
||||
all_headers["Authorization"] = "Bearer " + token
|
||||
|
||||
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as reply:
|
||||
response = Response(reply.status, reply.headers, reply.read())
|
||||
except urllib.error.HTTPError as error:
|
||||
response = Response(error.code, error.headers, error.read())
|
||||
|
||||
expected = expected if isinstance(expected, tuple) else (expected,)
|
||||
check(response.status in expected,
|
||||
f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}")
|
||||
request_id = response.headers.get("x-request-id")
|
||||
check(request_id, f"{method} {path}: no X-Request-Id header")
|
||||
check(response.instance, f"{method} {path}: no X-Instance-Id header")
|
||||
if response.status >= 400:
|
||||
error = (response.json or {}).get("error") or {}
|
||||
check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format")
|
||||
check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header")
|
||||
if response.status == 401:
|
||||
check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate")
|
||||
return response
|
||||
|
||||
|
||||
def try_call(method, path, *, body=None, token=None, headers=None, timeout=2):
|
||||
"""Запрос без проверок и без исключений — для нагрузки во время отказа.
|
||||
|
||||
Возвращает Response или None, если соединение оборвалось / вышел timeout.
|
||||
"""
|
||||
all_headers = dict(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
all_headers["Content-Type"] = "application/json"
|
||||
if token is not None:
|
||||
all_headers["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as reply:
|
||||
return Response(reply.status, reply.headers, reply.read())
|
||||
except urllib.error.HTTPError as error:
|
||||
try:
|
||||
return Response(error.code, error.headers, error.read())
|
||||
except OSError:
|
||||
return None
|
||||
except (urllib.error.URLError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
"""Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8."""
|
||||
return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
||||
|
||||
|
||||
def login(username, password, expected=201):
|
||||
return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)})
|
||||
|
||||
|
||||
def new_user(prefix="user", password=PASSWORD):
|
||||
"""Регистрирует пользователя и входит. Возвращает dict: id, username, password, token."""
|
||||
username = f"{prefix}_{uuid.uuid4().hex[:10]}"
|
||||
user = call("POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": prefix.title(), "password": password}).json
|
||||
session = login(username, password).json
|
||||
return {**user, "password": password, "token": session["token"]}
|
||||
|
||||
|
||||
def new_chat(owner, *others, title="Test chat"):
|
||||
member_ids = [owner["id"]] + [user["id"] for user in others]
|
||||
return call("POST", "/api/v1/chats", 201, token=owner["token"],
|
||||
body={"title": title, "member_ids": member_ids}).json
|
||||
|
||||
|
||||
def new_key():
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def send(user, chat_id, text, expected=201, key=None):
|
||||
"""POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный."""
|
||||
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
|
||||
body={"text": text}, headers={"Idempotency-Key": key or new_key()})
|
||||
|
||||
|
||||
def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None):
|
||||
"""POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся."""
|
||||
body = {"text": text}
|
||||
if attachment_ids is not None:
|
||||
body["attachment_ids"] = attachment_ids
|
||||
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
|
||||
body=body, headers={"Idempotency-Key": key or new_key()})
|
||||
|
||||
|
||||
def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=201, timeout=60):
|
||||
"""POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра."""
|
||||
query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="")
|
||||
headers = {"Content-Type": content_type} if content_type is not None else {}
|
||||
return call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data,
|
||||
token=user["token"] if user else None, headers=headers, timeout=timeout)
|
||||
|
||||
|
||||
def read_history(user, chat_id, limit=100):
|
||||
"""Вся история чата по страницам. Возвращает (сообщения, число_страниц)."""
|
||||
items, pages, cursor = [], 0, None
|
||||
while True:
|
||||
query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "")
|
||||
page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json
|
||||
items += page["items"]
|
||||
pages += 1
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return items, pages
|
||||
|
||||
|
||||
def wait_ready(seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply:
|
||||
if reply.status == 200:
|
||||
return
|
||||
except (urllib.error.URLError, OSError):
|
||||
pass
|
||||
check(time.monotonic() < deadline, f"API is not ready after {seconds} s")
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
"""docker compose <args> в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам."""
|
||||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def compose_output(*args, env=None):
|
||||
"""docker compose <args> и его вывод (для logs, ps)."""
|
||||
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env)
|
||||
return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def docker(*args, check_exit=True):
|
||||
"""docker <args>, возвращает stdout."""
|
||||
result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit)
|
||||
return result.stdout.decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def api_containers(include_stopped=False):
|
||||
"""Полные ID контейнеров сервиса api (по умолчанию только запущенные)."""
|
||||
args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"]
|
||||
return sorted(compose_output(*args).split())
|
||||
|
||||
|
||||
def health(container):
|
||||
"""Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''."""
|
||||
return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container,
|
||||
check_exit=False)
|
||||
|
||||
|
||||
def lb_servers():
|
||||
"""Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}."""
|
||||
with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply:
|
||||
rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace"))))
|
||||
header = [name.lstrip("# ") for name in rows[0]]
|
||||
servers = {}
|
||||
for row in rows[1:]:
|
||||
item = dict(zip(header, row))
|
||||
if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"):
|
||||
servers[item["svname"]] = item.get("status", "")
|
||||
return servers
|
||||
|
||||
|
||||
def lb_up_count():
|
||||
"""Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP)."""
|
||||
return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP"))
|
||||
|
||||
|
||||
def seen_instances(token, requests=20):
|
||||
"""Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me."""
|
||||
return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)}
|
||||
|
||||
|
||||
def wait_balanced(replicas, seconds=60):
|
||||
"""Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
running = api_containers()
|
||||
healthy = [container for container in running if health(container) == "healthy"]
|
||||
try:
|
||||
up = lb_up_count()
|
||||
except OSError:
|
||||
up = 0
|
||||
if len(healthy) >= replicas and up >= replicas:
|
||||
return running
|
||||
check(time.monotonic() < deadline,
|
||||
f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}")
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def redis_cli(*args):
|
||||
"""redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод."""
|
||||
command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args)
|
||||
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command],
|
||||
cwd=ROOT, capture_output=True, check=True)
|
||||
return result.stdout.decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def session_key(token):
|
||||
"""Ключ сессии в Redis, как его считает auth.py: session:<SHA-256 токена>."""
|
||||
return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def psql(sql):
|
||||
"""Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки).
|
||||
|
||||
Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя.
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "postgres",
|
||||
"psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql],
|
||||
cwd=ROOT, capture_output=True, check=False,
|
||||
)
|
||||
output = result.stdout.decode("utf-8", "replace").strip()
|
||||
error = result.stderr.decode("utf-8", "replace").strip()
|
||||
return result.returncode, output, error
|
||||
|
||||
|
||||
def app_python(code, *args, service="api", env=None):
|
||||
"""Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API).
|
||||
|
||||
Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON).
|
||||
"""
|
||||
command = ["docker", "compose", "exec", "-T"]
|
||||
for name, value in (env or {}).items():
|
||||
command += ["-e", f"{name}={value}"]
|
||||
command += [service, "python", "-", *map(str, args)]
|
||||
result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False)
|
||||
output = result.stdout.decode("utf-8", "replace").strip()
|
||||
check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
|
||||
return output
|
||||
|
||||
|
||||
def janitor(*args, env=None):
|
||||
"""Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт."""
|
||||
command = ["docker", "compose", "exec", "-T"]
|
||||
for name, value in (env or {}).items():
|
||||
command += ["-e", f"{name}={value}"]
|
||||
command += ["janitor", "python", "janitor.py", "--once", *map(str, args)]
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False)
|
||||
output = result.stdout.decode("utf-8", "replace").strip()
|
||||
check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
|
||||
return json.loads(output.splitlines()[-1])
|
||||
|
||||
|
||||
def sql_value(query):
|
||||
"""Одно значение из PostgreSQL (psql -A -t)."""
|
||||
code, output, error = psql(query)
|
||||
check(code == 0, f"psql failed: {error[-300:]}")
|
||||
return output
|
||||
|
||||
|
||||
def run_tests(tests):
|
||||
"""Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал."""
|
||||
failed = 0
|
||||
for test in tests:
|
||||
try:
|
||||
test()
|
||||
except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста
|
||||
failed += 1
|
||||
print(f"FAIL: {test.__name__}: {exc}")
|
||||
print(f"{len(tests) - failed} passed, {failed} failed")
|
||||
raise SystemExit(1 if failed else 0)
|
||||
@@ -0,0 +1,16 @@
|
||||
|
||||
from common import call, compose_output, new_user
|
||||
|
||||
|
||||
def main():
|
||||
user = new_user("demo")
|
||||
print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:")
|
||||
for number in range(1, 9):
|
||||
reply = call("GET", "/api/v1/users/me", 200, token=user["token"])
|
||||
print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}")
|
||||
print()
|
||||
print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
|
||||
import subprocess
|
||||
|
||||
from common import ROOT
|
||||
|
||||
OUTPUT = ROOT / "evidence" / "explain-history.txt"
|
||||
|
||||
|
||||
def main():
|
||||
sql = (ROOT / "student-tests" / "explain.sql").read_bytes()
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "postgres",
|
||||
"psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"],
|
||||
cwd=ROOT, input=sql, capture_output=True, check=False,
|
||||
)
|
||||
OUTPUT.parent.mkdir(exist_ok=True)
|
||||
OUTPUT.write_bytes(result.stdout + result.stderr)
|
||||
if result.returncode != 0:
|
||||
print(result.stderr.decode("utf-8", "replace"))
|
||||
raise SystemExit("FAIL: psql returned an error, see output above")
|
||||
for line in result.stdout.decode("utf-8", "replace").splitlines():
|
||||
if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line:
|
||||
print(line)
|
||||
print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
|
||||
\pset footer off
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE seed_chats AS
|
||||
SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size
|
||||
FROM generate_series(0, 50) AS n;
|
||||
CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id;
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash'
|
||||
FROM seed_user;
|
||||
INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats;
|
||||
INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'),
|
||||
timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second'
|
||||
FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g;
|
||||
ANALYZE messages;
|
||||
|
||||
SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset
|
||||
SELECT created_at AS cursor_time, id AS cursor_id FROM messages
|
||||
WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
|
||||
|
||||
\echo '=== Data volume'
|
||||
SELECT count(*) AS messages_total,
|
||||
count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat
|
||||
FROM messages;
|
||||
|
||||
\echo ''
|
||||
\echo '=== History index'
|
||||
SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history';
|
||||
|
||||
\echo ''
|
||||
\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat'
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
|
||||
FROM messages m
|
||||
WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY m.created_at, m.id LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat'
|
||||
ORDER BY created_at, id OFFSET 10000 LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)'
|
||||
DROP INDEX messages_history;
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
ROLLBACK;
|
||||
\echo ''
|
||||
\echo '=== ROLLBACK: test data removed, index is back'
|
||||
SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%';
|
||||
@@ -0,0 +1,129 @@
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced
|
||||
from loadgen import Load, summary
|
||||
|
||||
WARMUP_SECONDS = 3
|
||||
AFTER_SECONDS = 10
|
||||
OUTPUT = ROOT / "evidence" / "failover-metrics.json"
|
||||
|
||||
|
||||
class LbWatch:
|
||||
"""Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP)."""
|
||||
|
||||
def __init__(self):
|
||||
self.points, self.stopping = [], threading.Event()
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def _run(self):
|
||||
while not self.stopping.is_set():
|
||||
try:
|
||||
self.points.append((time.monotonic(), lb_up_count()))
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
|
||||
def start(self):
|
||||
self.thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.stopping.set()
|
||||
self.thread.join(timeout=3)
|
||||
|
||||
def first_drop(self, after, replicas):
|
||||
return next((t for t, up in self.points if t >= after and up < replicas), None)
|
||||
|
||||
|
||||
def measure(container, mode, replicas):
|
||||
wait_balanced(replicas)
|
||||
load, watch = Load(), LbWatch()
|
||||
watch.start()
|
||||
load.start()
|
||||
time.sleep(WARMUP_SECONDS)
|
||||
t0 = time.monotonic()
|
||||
if mode == "stop":
|
||||
docker("stop", "--time", "8", container)
|
||||
else:
|
||||
docker("kill", container)
|
||||
command_seconds = time.monotonic() - t0
|
||||
time.sleep(max(0.0, AFTER_SECONDS - command_seconds))
|
||||
end = t0 + AFTER_SECONDS
|
||||
load.stop()
|
||||
watch.stop()
|
||||
|
||||
errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]]
|
||||
errors_end = max((e[0] + e[5] for e in errors), default=None)
|
||||
drop = watch.first_drop(t0, replicas)
|
||||
served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]]
|
||||
target_hits = sum(1 for instance in served_after if instance and container.startswith(instance))
|
||||
integrity = load.verify()
|
||||
|
||||
started = time.monotonic()
|
||||
docker("start", container)
|
||||
wait_balanced(replicas)
|
||||
rejoin_seconds = time.monotonic() - started
|
||||
|
||||
return {
|
||||
"replica": container[:12],
|
||||
"mode": mode,
|
||||
"command_seconds": round(command_seconds, 2),
|
||||
"lb_excluded_after_seconds": round(drop - t0, 2) if drop else None,
|
||||
"errors": len(errors),
|
||||
"error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"],
|
||||
"errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0,
|
||||
"before": summary(load, t0 - WARMUP_SECONDS, t0),
|
||||
"after": summary(load, t0, end),
|
||||
"stopped_replica_answers_after_stop": target_hits,
|
||||
"integrity": integrity,
|
||||
"rejoin_seconds": round(rejoin_seconds, 2),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
modes = sys.argv[1:] or ["stop", "kill"]
|
||||
check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]")
|
||||
containers = api_containers()
|
||||
replicas = len(containers)
|
||||
check(replicas >= 2, "need at least 2 running api replicas")
|
||||
print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, "
|
||||
f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and "
|
||||
f"{AFTER_SECONDS} s after the failure")
|
||||
runs = []
|
||||
for container in containers:
|
||||
for mode in modes:
|
||||
print(f"--- {mode} {container[:12]} ...")
|
||||
run = measure(container, mode, replicas)
|
||||
runs.append(run)
|
||||
print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after "
|
||||
f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, "
|
||||
f"stopped after {run['errors_stopped_after_seconds']} s")
|
||||
print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; "
|
||||
f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; "
|
||||
f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}")
|
||||
print(f" confirmed {run['integrity']['confirmed_writes']}, in history "
|
||||
f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; "
|
||||
f"back in the balancer {run['rejoin_seconds']} s after docker start")
|
||||
check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered")
|
||||
check(run["after"]["write_rps"] > 0, "no successful writes after the failure")
|
||||
print()
|
||||
print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s")
|
||||
for r in runs:
|
||||
print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} "
|
||||
f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} "
|
||||
f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}")
|
||||
OUTPUT.parent.mkdir(exist_ok=True)
|
||||
OUTPUT.write_text(json.dumps({
|
||||
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS,
|
||||
"after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()},
|
||||
"runs": runs,
|
||||
}, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; "
|
||||
f"saved {OUTPUT.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
@@ -0,0 +1,93 @@
|
||||
"""Синтетические изображения для тестов вложений: только стандартная библиотека.
|
||||
|
||||
PNG собирается вручную (сигнатура, IHDR, IDAT, IEND), поэтому можно получить любой размер
|
||||
в пикселях и байтах, а также испортить конкретный байт. JPEG берётся из fixtures/.
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes:
|
||||
"""PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой).
|
||||
|
||||
pad_to — дополнить файл текстовым чанком tEXt после IDAT до точного размера в байтах.
|
||||
idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое).
|
||||
"""
|
||||
channels = 1 if gray else 3
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace)
|
||||
if idat is None:
|
||||
compressor = zlib.compressobj(6)
|
||||
parts = []
|
||||
# Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход.
|
||||
passes = [(width, height)] if not interlace else [
|
||||
((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7]
|
||||
for pass_width, pass_height in passes:
|
||||
if pass_width <= 0 or pass_height <= 0:
|
||||
continue
|
||||
row = b"\x00" + bytes(pass_width * channels)
|
||||
for _ in range(pass_height):
|
||||
parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row))
|
||||
parts.append(compressor.flush())
|
||||
idat = b"".join(parts)
|
||||
body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat)
|
||||
if pad_to is not None:
|
||||
need = pad_to - len(body) - 12 - 12 - 8 # заголовок чанка-заполнителя, IEND, ключ "Comment\0"
|
||||
assert need >= 0, "image is already larger than pad_to"
|
||||
body += chunk(b"tEXt", b"Comment\x00" + b"a" * need)
|
||||
return body + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def noise_png(target_bytes: int) -> bytes:
|
||||
"""Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями)."""
|
||||
side = int(((target_bytes - 4096) / 3) ** 0.5)
|
||||
return png(side, side, noise=True)
|
||||
|
||||
|
||||
def sample_png() -> bytes:
|
||||
return (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def sample_jpeg() -> bytes:
|
||||
return (FIXTURES / "sample.jpg").read_bytes()
|
||||
|
||||
|
||||
def progressive_jpeg() -> bytes:
|
||||
return (FIXTURES / "progressive.jpg").read_bytes()
|
||||
|
||||
|
||||
def flip_byte(data: bytes, position: int) -> bytes:
|
||||
return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:]
|
||||
|
||||
|
||||
GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"
|
||||
|
||||
|
||||
# --- сборка PNG из произвольных чанков (для проверок смысла чанков при верных CRC) ---
|
||||
|
||||
def ihdr(width, height, depth=8, color=2, interlace=0):
|
||||
return b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)
|
||||
|
||||
|
||||
def zero_idat(width, height, depth=8, channels=3):
|
||||
"""Сжатые пиксели: каждая строка = байт фильтра 0 + нули (индекс палитры 0 / чёрный)."""
|
||||
row_bytes = (width * depth * channels + 7) // 8
|
||||
return b"IDAT", zlib.compress((b"\x00" + bytes(row_bytes)) * height)
|
||||
|
||||
|
||||
def assemble(*chunks):
|
||||
"""PNG из чанков (kind, data) с ВЕРНЫМИ CRC; IEND добавляется, если не передан."""
|
||||
if not chunks or chunks[-1][0] != b"IEND":
|
||||
chunks = chunks + ((b"IEND", b""),)
|
||||
return b"\x89PNG\r\n\x1a\n" + b"".join(chunk(kind, data) for kind, data in chunks)
|
||||
@@ -0,0 +1,102 @@
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
|
||||
from common import check, new_chat, new_key, new_user, read_history, try_call
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, writers=4, timeout=2):
|
||||
self.alice, self.bob = new_user("load_a"), new_user("load_b")
|
||||
self.chat = new_chat(self.alice, self.bob, title="load")
|
||||
self.writers, self.timeout = writers, timeout
|
||||
self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с)
|
||||
self.confirmed = [] # сообщения, на которые пришёл 201
|
||||
self.lock = threading.Lock()
|
||||
self.stopping = threading.Event()
|
||||
self.threads = []
|
||||
|
||||
def _record(self, kind, started, reply, ok):
|
||||
code = reply.status if reply is not None else "conn"
|
||||
instance = reply.instance if reply is not None else None
|
||||
with self.lock:
|
||||
self.events.append((started, kind, ok, code, instance, time.monotonic() - started))
|
||||
|
||||
def _writer(self, number):
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||
sent = 0
|
||||
key, text = new_key(), f"load-{number}-{sent}"
|
||||
while not self.stopping.is_set():
|
||||
started = time.monotonic()
|
||||
reply = try_call("POST", path, body={"text": text}, token=self.alice["token"],
|
||||
headers={"Idempotency-Key": key}, timeout=self.timeout)
|
||||
ok = reply is not None and reply.status == 201
|
||||
self._record("write", started, reply, ok)
|
||||
if ok:
|
||||
with self.lock:
|
||||
self.confirmed.append(reply.json)
|
||||
sent += 1
|
||||
key, text = new_key(), f"load-{number}-{sent}"
|
||||
else:
|
||||
time.sleep(0.05) # и повторить тот же text с тем же ключом
|
||||
|
||||
def _reader(self):
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20"
|
||||
while not self.stopping.is_set():
|
||||
started = time.monotonic()
|
||||
reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout)
|
||||
self._record("read", started, reply, reply is not None and reply.status == 200)
|
||||
time.sleep(0.05)
|
||||
|
||||
def start(self):
|
||||
self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)]
|
||||
self.threads.append(threading.Thread(target=self._reader, daemon=True))
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.stopping.set()
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=self.timeout + 5)
|
||||
|
||||
def window(self, start, end, kind=None):
|
||||
with self.lock:
|
||||
return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)]
|
||||
|
||||
def verify(self):
|
||||
"""Все подтверждённые сообщения есть в истории, id и тексты не повторяются."""
|
||||
history, _ = read_history(self.bob, self.chat["id"])
|
||||
ids = [message["id"] for message in history]
|
||||
texts = [message["text"] for message in history]
|
||||
by_id = {message["id"]: message for message in history}
|
||||
check(len(ids) == len(set(ids)), "duplicated message ids in history")
|
||||
check(len(texts) == len(set(texts)), "a retried request created a duplicate message")
|
||||
lost = [m for m in self.confirmed if by_id.get(m["id"]) != m]
|
||||
check(not lost, f"{len(lost)} confirmed messages are lost or changed")
|
||||
return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0}
|
||||
|
||||
|
||||
def summary(load, start, end):
|
||||
"""Успешные RPS, ошибки и задержки на отрезке [start, end)."""
|
||||
events = load.window(start, end)
|
||||
writes = [e for e in events if e[1] == "write"]
|
||||
ok_writes = sorted(e[5] for e in writes if e[2])
|
||||
errors = [e for e in events if not e[2]]
|
||||
seconds = max(end - start, 1e-6)
|
||||
|
||||
def percentile(p):
|
||||
return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None
|
||||
|
||||
codes = {}
|
||||
for e in errors:
|
||||
codes[str(e[3])] = codes.get(str(e[3]), 0) + 1
|
||||
return {
|
||||
"seconds": round(seconds, 2),
|
||||
"write_rps": round(len(ok_writes) / seconds, 1),
|
||||
"reads": len([e for e in events if e[1] == "read"]),
|
||||
"errors": len(errors),
|
||||
"error_codes": codes,
|
||||
"write_p50_ms": percentile(0.50),
|
||||
"write_p95_ms": percentile(0.95),
|
||||
"write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None,
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import subprocess
|
||||
|
||||
from common import ROOT, login, new_user, redis_cli, session_key
|
||||
|
||||
|
||||
def without_password():
|
||||
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"],
|
||||
cwd=ROOT, capture_output=True, check=False)
|
||||
return (result.stdout + result.stderr).decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def main():
|
||||
user = new_user("info")
|
||||
token = login(user["username"], user["password"]).json["token"]
|
||||
key = session_key(token)
|
||||
print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}")
|
||||
print(f"TTL of the key, seconds: {redis_cli('ttl', key)}")
|
||||
keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines()
|
||||
print(f"sessions in Redis: {len(keys)}")
|
||||
for name in ("appendonly", "appendfsync", "maxmemory-policy"):
|
||||
print("config", " = ".join(redis_cli("config", "get", name).split()))
|
||||
for line in redis_cli("info", "persistence").splitlines():
|
||||
if line.startswith(("aof_enabled", "aof_last_write_status")):
|
||||
print(line)
|
||||
print(f"without password: {without_password()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced
|
||||
from loadgen import Load, summary
|
||||
|
||||
OUTPUT = ROOT / "evidence" / "rolling-update.json"
|
||||
|
||||
|
||||
def env_of(container):
|
||||
lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines()
|
||||
return dict(line.split("=", 1) for line in lines if "=" in line)
|
||||
|
||||
|
||||
def wait_new_replica(container, up_needed, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while health(container) != "healthy" or lb_up_count() < up_needed:
|
||||
check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main():
|
||||
old = api_containers()
|
||||
replicas = len(old)
|
||||
check(replicas >= 2, "need at least 2 running api replicas")
|
||||
wait_balanced(replicas)
|
||||
old_version = env_of(old[0]).get("APP_VERSION")
|
||||
version = time.strftime("rolling-%Y%m%d-%H%M%S")
|
||||
env = dict(os.environ, APP_VERSION=version)
|
||||
|
||||
print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}")
|
||||
started = time.monotonic()
|
||||
compose("build", "api", "migrate", env=env)
|
||||
print(f"image built in {time.monotonic() - started:.1f} s")
|
||||
migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env)
|
||||
print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1]
|
||||
.split("messenger.db: ")[-1])
|
||||
|
||||
load = Load()
|
||||
load.start()
|
||||
time.sleep(3)
|
||||
t0 = time.monotonic()
|
||||
steps = []
|
||||
for container in old:
|
||||
step_start = time.monotonic()
|
||||
before = set(api_containers())
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env)
|
||||
new = sorted(set(api_containers()) - before)
|
||||
check(len(new) == 1, f"expected one new container, got {len(new)}")
|
||||
wait_new_replica(new[0], replicas + 1)
|
||||
ready = time.monotonic() - step_start
|
||||
docker("stop", "--time", "15", container)
|
||||
docker("rm", container)
|
||||
steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2),
|
||||
"step_seconds": round(time.monotonic() - step_start, 2)})
|
||||
print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, "
|
||||
f"step {steps[-1]['step_seconds']} s")
|
||||
t1 = time.monotonic()
|
||||
time.sleep(3)
|
||||
load.stop()
|
||||
|
||||
final = api_containers()
|
||||
versions = sorted({env_of(c).get("APP_VERSION") for c in final})
|
||||
integrity = load.verify()
|
||||
during = summary(load, t0, t1)
|
||||
before_stats = summary(load, t0 - 3, t0)
|
||||
print(f"now: replicas {[c[:12] for c in final]}, versions {versions}")
|
||||
print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; "
|
||||
f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, "
|
||||
f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s")
|
||||
print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, "
|
||||
f"lost 0, duplicates 0")
|
||||
check(len(final) == replicas and not set(final) & set(old), "old replicas are still running")
|
||||
check(versions == [version], "not every replica runs the new version")
|
||||
OUTPUT.parent.mkdir(exist_ok=True)
|
||||
OUTPUT.write_text(json.dumps({
|
||||
"from_version": old_version, "to_version": version, "replicas": replicas,
|
||||
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2},
|
||||
"steps": steps, "update_seconds": round(t1 - t0, 2),
|
||||
"before": before_stats, "during": during, "integrity": integrity,
|
||||
}, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"PASS: all {replicas} replicas replaced one by one under load, "
|
||||
f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced
|
||||
|
||||
HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
||||
|
||||
|
||||
def ps():
|
||||
return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()
|
||||
|
||||
|
||||
def aim(wanted):
|
||||
"""Для 2 реплик: последним перед тестом должна ответить реплика из wanted."""
|
||||
probe = call("GET", "/health/live", 200).instance
|
||||
if not any(container.startswith(probe) for container in wanted):
|
||||
call("GET", "/health/live", 200) # теперь последней ответила другая реплика
|
||||
|
||||
|
||||
def main():
|
||||
containers = api_containers()
|
||||
check(len(containers) >= 2, "need at least 2 running api replicas")
|
||||
print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}")
|
||||
stopped_once = set()
|
||||
for attempt in range(1, 7):
|
||||
wait_balanced(len(containers))
|
||||
if len(containers) == 2:
|
||||
aim(set(containers) - stopped_once)
|
||||
print(f"=== run {attempt}: api containers before")
|
||||
print(ps())
|
||||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
|
||||
result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover",
|
||||
"--action-script", str(HOOK), "--base-url", BASE_URL],
|
||||
cwd=ROOT, capture_output=True, env=env)
|
||||
print((result.stdout + result.stderr).decode("utf-8", "replace").strip())
|
||||
check(result.returncode == 0, "public failover test failed")
|
||||
print("api containers after")
|
||||
print(ps())
|
||||
stopped = [container for container in containers if container not in api_containers()]
|
||||
check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}")
|
||||
print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); "
|
||||
f"other api containers kept running, postgres/redis/lb were not touched")
|
||||
stopped_once.add(stopped[0])
|
||||
docker("start", stopped[0])
|
||||
wait_balanced(len(containers))
|
||||
print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy")
|
||||
if stopped_once == set(containers):
|
||||
break
|
||||
check(stopped_once == set(containers), "not every replica was stopped by the public test")
|
||||
print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from common import BASE_URL, ROOT, check, compose_output, docker
|
||||
|
||||
HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh")
|
||||
|
||||
|
||||
def started():
|
||||
containers = compose_output("ps", "-q").split()
|
||||
lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines()
|
||||
return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)}
|
||||
|
||||
|
||||
def main():
|
||||
before = started()
|
||||
print("before: container, ID, started at")
|
||||
for name, (cid, at) in sorted(before.items()):
|
||||
print(f" {name:<28} {cid[:12]} {at}")
|
||||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
|
||||
begin = time.monotonic()
|
||||
result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence",
|
||||
"--action-script", str(HOOK), "--base-url", BASE_URL],
|
||||
cwd=ROOT, capture_output=True, env=env)
|
||||
seconds = time.monotonic() - begin
|
||||
print((result.stdout + result.stderr).decode("utf-8", "replace").strip())
|
||||
check(result.returncode == 0, "public persistence test failed")
|
||||
after = started()
|
||||
print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?")
|
||||
restarted = []
|
||||
for name, (cid, at) in sorted(after.items()):
|
||||
same_id = before.get(name, ("", ""))[0] == cid
|
||||
new_start = before.get(name, ("", ""))[1] != at
|
||||
print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}")
|
||||
check(same_id, f"{name}: container was recreated, volumes/IDs must stay")
|
||||
if new_start:
|
||||
restarted.append(name.split("-")[-2])
|
||||
for service in ("postgres", "redis", "s3", "api", "janitor", "lb"):
|
||||
check(service in restarted, f"{service} was not restarted by the hook")
|
||||
print("PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, "
|
||||
"new start time); data, the attachment bytes and the old token survived")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,258 @@
|
||||
import base64
|
||||
import datetime as dt
|
||||
import uuid
|
||||
|
||||
from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send
|
||||
|
||||
|
||||
def parse_time(value):
|
||||
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
|
||||
def test_duplicate_username():
|
||||
body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD}
|
||||
call("POST", "/api/v1/users", 201, body=body)
|
||||
call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user"))
|
||||
print("PASS: duplicate username returns 409")
|
||||
|
||||
|
||||
def test_invalid_json():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
call("POST", "/api/v1/users", 400, raw=b'{"username":')
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json",
|
||||
headers={"Idempotency-Key": new_key()})
|
||||
print("PASS: invalid JSON returns 400")
|
||||
|
||||
|
||||
def test_wrong_fields():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD})
|
||||
call("POST", "/api/v1/users", 400,
|
||||
body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True})
|
||||
# Отправителя нельзя подставить: он берётся из сессии.
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
|
||||
body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()})
|
||||
print("PASS: wrong and extra fields return 400")
|
||||
|
||||
|
||||
def test_foreign_chat():
|
||||
alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve")
|
||||
chat = new_chat(alice, bob)
|
||||
call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"])
|
||||
call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"])
|
||||
send(eve, chat["id"], "not my chat", expected=404)
|
||||
# Несуществующий чат выглядит так же, как чужой.
|
||||
call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"])
|
||||
print("PASS: foreign chat returns 404")
|
||||
|
||||
|
||||
def test_empty_message():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
for text in ("", " ", "\n\t"):
|
||||
send(alice, chat["id"], text, expected=400)
|
||||
send(alice, chat["id"], "я" * 4001, expected=400)
|
||||
send(alice, chat["id"], "я" * 4000)
|
||||
print("PASS: empty and too long messages return 400, 4000 chars is ok")
|
||||
|
||||
|
||||
def test_pagination():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)]
|
||||
|
||||
history, pages = read_history(bob, chat["id"], limit=2)
|
||||
check(pages == 4, f"expected 4 pages, got {pages}")
|
||||
check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages")
|
||||
|
||||
other = new_chat(alice)
|
||||
for i in range(3):
|
||||
send(alice, other["id"], f"Other {i}")
|
||||
first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json
|
||||
call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"])
|
||||
call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"])
|
||||
print("PASS: pagination works across multiple pages, foreign cursor returns 400")
|
||||
|
||||
|
||||
def test_empty_cursor():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
send(alice, chat["id"], "one message")
|
||||
for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"):
|
||||
for query in ("?cursor=", "?cursor", "?limit=10&cursor="):
|
||||
call("GET", path + query, 400, token=alice["token"])
|
||||
call("GET", path, 200, token=alice["token"])
|
||||
print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page")
|
||||
|
||||
|
||||
def test_chat_list_pagination():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)]
|
||||
seen, cursor = [], None
|
||||
while True:
|
||||
query = "?limit=1" + (f"&cursor={cursor}" if cursor else "")
|
||||
page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json
|
||||
seen += [chat["id"] for chat in page["items"]]
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
check(seen == created, "chat list pages are wrong")
|
||||
print("PASS: chat list pagination")
|
||||
|
||||
|
||||
def test_unicode():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob, title="Чат 👋")
|
||||
text = "Привет 👋 你好"
|
||||
check(send(alice, chat["id"], text).json["text"] == text, "text changed on write")
|
||||
history, _ = read_history(bob, chat["id"])
|
||||
check(history[-1]["text"] == text, "text changed on read")
|
||||
print("PASS: unicode text is stored as is")
|
||||
|
||||
|
||||
def test_nul_character_is_stored():
|
||||
name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world"
|
||||
body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"}
|
||||
user = call("POST", "/api/v1/users", 201, body=body).json
|
||||
check(user["display_name"] == name, "display_name with U+0000 changed")
|
||||
token = login(body["username"], body["password"]).json["token"]
|
||||
check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read")
|
||||
chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json
|
||||
check(chat["title"] == title, "title with U+0000 changed")
|
||||
check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read")
|
||||
sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text},
|
||||
headers={"Idempotency-Key": new_key()}).json
|
||||
history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"]
|
||||
check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed")
|
||||
login("nul\u0000user", PASSWORD, expected=401)
|
||||
print("PASS: strings with U+0000 are stored and returned as is")
|
||||
|
||||
|
||||
|
||||
def test_user_has_no_password():
|
||||
body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD}
|
||||
user = call("POST", "/api/v1/users", 201, body=body).json
|
||||
check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}")
|
||||
print("PASS: user response has no password or hash")
|
||||
|
||||
|
||||
def test_password_rules():
|
||||
def register(password, expected):
|
||||
body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password}
|
||||
call("POST", "/api/v1/users", expected, body=body)
|
||||
|
||||
register("x" * 11, 400)
|
||||
register("x" * 129, 400)
|
||||
register(" " * 12, 400)
|
||||
register("x" * 12, 201)
|
||||
register("я" * 128, 201)
|
||||
print("PASS: password length 12..128, blank password is rejected")
|
||||
|
||||
|
||||
def test_login_gives_token():
|
||||
username = "lg_" + uuid.uuid4().hex[:10]
|
||||
user = call("POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": "Login", "password": PASSWORD}).json
|
||||
reply = login(username, PASSWORD)
|
||||
session = reply.json
|
||||
check(session["token_type"] == "Bearer", "token_type must be Bearer")
|
||||
check(session["user"] == user, "session user differs from registered user")
|
||||
check(len(session["token"]) >= 43, "token is too short")
|
||||
check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque")
|
||||
check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached")
|
||||
left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds()
|
||||
check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s")
|
||||
me = call("GET", "/api/v1/users/me", token=session["token"]).json
|
||||
check(me == user, "/users/me returned another user")
|
||||
print("PASS: Basic login returns opaque Bearer token with expiry")
|
||||
|
||||
|
||||
def test_wrong_password():
|
||||
user = new_user("wp")
|
||||
wrong = login(user["username"], "Wrong-password-000", expected=401)
|
||||
unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401)
|
||||
check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic")
|
||||
check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames")
|
||||
print("PASS: wrong password and unknown user both return the same 401")
|
||||
|
||||
|
||||
def test_malformed_basic():
|
||||
no_colon = base64.b64encode(b"just_username").decode()
|
||||
for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"):
|
||||
headers = {} if header is None else {"Authorization": header}
|
||||
call("POST", "/api/v1/auth/sessions", 401, headers=headers)
|
||||
print("PASS: malformed Basic header returns 401")
|
||||
|
||||
|
||||
def test_password_with_colon_and_unicode():
|
||||
password = "пароль:с:двоеточием 👋"
|
||||
user = new_user("colon", password=password)
|
||||
call("GET", "/api/v1/users/me", token=user["token"])
|
||||
print("PASS: password with colon and unicode works")
|
||||
|
||||
|
||||
def test_bearer_required():
|
||||
user = new_user("br")
|
||||
for headers in ({}, {"Authorization": basic(user["username"], user["password"])},
|
||||
{"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43},
|
||||
{"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}):
|
||||
reply = call("GET", "/api/v1/users/me", 401, headers=headers)
|
||||
check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer")
|
||||
call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]})
|
||||
print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401")
|
||||
|
||||
|
||||
def test_x_user_id_ignored():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json
|
||||
check(me["id"] == alice["id"], "X-User-Id changed the current user")
|
||||
print("PASS: X-User-Id does not change the current user")
|
||||
|
||||
|
||||
def test_auth_before_body():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json")
|
||||
call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401)
|
||||
print("PASS: 401 comes before 400 and 404")
|
||||
|
||||
|
||||
def test_logout():
|
||||
user = new_user("lo")
|
||||
second = login(user["username"], user["password"]).json["token"]
|
||||
call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"])
|
||||
call("GET", "/api/v1/users/me", 401, token=user["token"])
|
||||
call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"])
|
||||
call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива
|
||||
again = login(user["username"], user["password"]).json["token"]
|
||||
check(again != user["token"], "new login must give a new token")
|
||||
call("GET", "/api/v1/users/me", 200, token=again)
|
||||
print("PASS: logout revokes only the current session")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([
|
||||
test_duplicate_username,
|
||||
test_invalid_json,
|
||||
test_wrong_fields,
|
||||
test_foreign_chat,
|
||||
test_empty_message,
|
||||
test_pagination,
|
||||
test_empty_cursor,
|
||||
test_chat_list_pagination,
|
||||
test_unicode,
|
||||
test_nul_character_is_stored,
|
||||
test_user_has_no_password,
|
||||
test_password_rules,
|
||||
test_login_gives_token,
|
||||
test_wrong_password,
|
||||
test_malformed_basic,
|
||||
test_password_with_colon_and_unicode,
|
||||
test_bearer_required,
|
||||
test_x_user_id_ignored,
|
||||
test_auth_before_body,
|
||||
test_logout,
|
||||
])
|
||||
@@ -0,0 +1,336 @@
|
||||
"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ.
|
||||
|
||||
Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа:
|
||||
пользовательская ошибка не должна становиться 500.
|
||||
"""
|
||||
import hashlib
|
||||
import http.client
|
||||
import socket
|
||||
import struct
|
||||
import urllib.parse
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
import images
|
||||
from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload
|
||||
|
||||
URL = urllib.parse.urlsplit(BASE_URL)
|
||||
|
||||
|
||||
def setup():
|
||||
alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve")
|
||||
return alice, bob, eve, new_chat(alice, bob)
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def check_error(reply, code):
|
||||
check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}")
|
||||
|
||||
|
||||
def test_png_and_jpeg_roundtrip():
|
||||
alice, bob, _, chat = setup()
|
||||
for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"),
|
||||
(images.sample_jpeg(), "image/jpeg", "sample.jpg"),
|
||||
(images.progressive_jpeg(), "image/jpeg", "progressive.jpg"),
|
||||
(images.png(33, 17, interlace=1), "image/png", "interlaced.png"),
|
||||
(images.png(1, 1, gray=True), "image/png", "1x1.png")):
|
||||
att = upload(alice, chat["id"], data, ctype, name).json
|
||||
check(att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None, "not ready")
|
||||
check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ")
|
||||
check(att["content_type"] == ctype and att["filename"] == name, "metadata differs")
|
||||
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json
|
||||
check(meta == att, "GET metadata differs from POST response")
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
check(got.data == data, "downloaded bytes differ")
|
||||
check(got.headers["content-type"] == ctype, "wrong Content-Type on download")
|
||||
check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length")
|
||||
check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing")
|
||||
print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes")
|
||||
print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte")
|
||||
|
||||
|
||||
def test_size_limit_exact_boundary():
|
||||
alice, _, _, chat = setup()
|
||||
exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES)
|
||||
check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB")
|
||||
att = upload(alice, chat["id"], exact).json
|
||||
check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed")
|
||||
print(f"exactly {len(exact)} bytes (10 MiB) -> 201")
|
||||
over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1)
|
||||
reply = upload(alice, chat["id"], over, expected=413)
|
||||
check_error(reply, "payload_too_large")
|
||||
print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body")
|
||||
print("PASS: the 10 MiB limit is exact")
|
||||
|
||||
|
||||
def chunked_upload(user, chat_id, total, piece=64 * 1024):
|
||||
"""Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении."""
|
||||
conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60)
|
||||
|
||||
def body():
|
||||
sent = 0
|
||||
head = images.sample_png()[:64]
|
||||
yield head
|
||||
sent += len(head)
|
||||
while sent < total:
|
||||
part = bytes(min(piece, total - sent))
|
||||
sent += len(part)
|
||||
yield part
|
||||
try:
|
||||
conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(),
|
||||
headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]},
|
||||
encode_chunked=True)
|
||||
reply = conn.getresponse()
|
||||
return reply.status, reply.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_size_limit_streaming():
|
||||
alice, _, _, chat = setup()
|
||||
status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024)
|
||||
check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}")
|
||||
print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed")
|
||||
call("GET", "/health/ready", 200)
|
||||
print("PASS: oversized streamed body is cut off, the API keeps working")
|
||||
|
||||
|
||||
def test_pixel_limit():
|
||||
alice, _, _, chat = setup()
|
||||
at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей
|
||||
att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json
|
||||
print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 201 {att['id'][:8]}")
|
||||
over = images.png(8001, 5000, gray=True)
|
||||
check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large")
|
||||
print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413")
|
||||
bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01")
|
||||
check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large")
|
||||
print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded")
|
||||
print("PASS: pixel limit is checked before decompression")
|
||||
|
||||
|
||||
def test_damaged_images():
|
||||
alice, _, _, chat = setup()
|
||||
png, jpeg = images.sample_png(), images.sample_jpeg()
|
||||
cases = [
|
||||
("PNG cut in half", png[:len(png) // 2], "image/png"),
|
||||
("PNG without IEND", png[:-12], "image/png"),
|
||||
("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"),
|
||||
("PNG with garbage after IEND", png + b"garbage", "image/png"),
|
||||
("PNG signature only", png[:8], "image/png"),
|
||||
("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"),
|
||||
("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"),
|
||||
("JPEG header only", jpeg[:20], "image/jpeg"),
|
||||
("random bytes", uuid.uuid4().bytes * 100, "image/png"),
|
||||
("plain text", "это не картинка".encode(), "image/jpeg"),
|
||||
]
|
||||
for name, data, ctype in cases:
|
||||
reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415)
|
||||
check_error(reply, "unsupported_media_type")
|
||||
print(f"{name} ({len(data)} bytes, {ctype}) -> 415")
|
||||
check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request")
|
||||
print("empty body -> 400")
|
||||
print("PASS: damaged and foreign files are rejected with 415, never 500")
|
||||
|
||||
|
||||
def test_png_chunk_semantics():
|
||||
"""CRC у всех чанков верные, но содержимое нарушает спецификацию PNG -> 415, а не 201.
|
||||
|
||||
Регрессия: PNG 1x1, глубина 8, палитровый, PLTE из 771 байта (257 записей) раньше получал 201 ready,
|
||||
хотя Pillow его отклоняет (invalid palette size).
|
||||
"""
|
||||
alice, bob, _, chat = setup()
|
||||
from images import assemble, ihdr, zero_idat
|
||||
pal = lambda entries: (b"PLTE", bytes(3 * entries))
|
||||
idat_p = zero_idat(1, 1, 8, 1)
|
||||
bad = [
|
||||
("PLTE 771 bytes = 257 entries (review case)",
|
||||
assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(771)), (b"IDAT", zlib.compress(b"\0\0")))),
|
||||
("PLTE length not a multiple of 3", assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(4)), idat_p)),
|
||||
("empty PLTE", assemble(ihdr(1, 1, 8, 3), (b"PLTE", b""), idat_p)),
|
||||
("palette larger than bit depth allows (depth 1, 3 entries)",
|
||||
assemble(ihdr(1, 1, 1, 3), pal(3), zero_idat(1, 1, 1, 1))),
|
||||
("palette image without PLTE", assemble(ihdr(1, 1, 8, 3), idat_p)),
|
||||
("PLTE in a grayscale image", assemble(ihdr(1, 1, 8, 0), pal(2), zero_idat(1, 1, 8, 1))),
|
||||
("PLTE after IDAT", assemble(ihdr(1, 1, 8, 3), idat_p, pal(2))),
|
||||
("two PLTE chunks", assemble(ihdr(1, 1, 8, 3), pal(2), pal(2), idat_p)),
|
||||
("tRNS longer than the palette", assemble(ihdr(1, 1, 8, 3), pal(2), (b"tRNS", bytes(3)), idat_p)),
|
||||
("tRNS before PLTE", assemble(ihdr(1, 1, 8, 3), (b"tRNS", bytes(1)), pal(2), idat_p)),
|
||||
("tRNS of wrong size for RGB (4 bytes instead of 6)",
|
||||
assemble(ihdr(1, 1, 8, 2), (b"tRNS", bytes(4)), zero_idat(1, 1, 8, 3))),
|
||||
("tRNS in an image with alpha channel",
|
||||
assemble(ihdr(1, 1, 8, 6), (b"tRNS", bytes(2)), zero_idat(1, 1, 8, 4))),
|
||||
("IEND with data", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"IEND", b"x"))),
|
||||
("unknown critical chunk", assemble(ihdr(1, 1, 8, 2), (b"ABCD", b""), zero_idat(1, 1, 8, 3))),
|
||||
("IDAT chunks not consecutive",
|
||||
assemble(ihdr(1, 2, 8, 2), (b"IDAT", zero_idat(1, 2, 8, 3)[1][:5]), (b"tEXt", b"k\0v"),
|
||||
(b"IDAT", zero_idat(1, 2, 8, 3)[1][5:]))),
|
||||
("gAMA after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"gAMA", bytes(4)))),
|
||||
("gAMA of 2 bytes instead of 4", assemble(ihdr(1, 1, 8, 2), (b"gAMA", bytes(2)), zero_idat(1, 1, 8, 3))),
|
||||
("3 MiB metadata chunk before IDAT", assemble(ihdr(1, 1, 8, 2), (b"prVt", bytes(3 * 1024 * 1024)),
|
||||
zero_idat(1, 1, 8, 3))),
|
||||
("private chunk after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"prVt", bytes(100)))),
|
||||
("animated PNG frame (fdAT)", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"fdAT", bytes(10)))),
|
||||
("tIME of 2 bytes instead of 7", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tIME", bytes(2)))),
|
||||
("tEXt without keyword separator", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tEXt", b"nokey"))),
|
||||
("zTXt with broken zlib after IDAT",
|
||||
assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0garbage"))),
|
||||
("zTXt text bomb: 50 MB of text in ~50 KB after IDAT",
|
||||
assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0" + zlib.compress(bytes(50_000_000))))),
|
||||
]
|
||||
for name, data in bad:
|
||||
check_error(upload(alice, chat["id"], data, filename="bad.png", expected=415), "unsupported_media_type")
|
||||
print(f"{name} ({len(data)} bytes, valid CRCs) -> 415")
|
||||
good = [
|
||||
("palette 256 entries + tRNS + tEXt and tIME after IDAT",
|
||||
assemble(ihdr(2, 2, 8, 3), pal(256), (b"tRNS", bytes(256)), zero_idat(2, 2, 8, 1),
|
||||
(b"tEXt", b"Comment\0ok"), (b"tIME", struct.pack(">HBBBBB", 2026, 10, 22, 12, 0, 0)))),
|
||||
("palette depth 2 with 4 entries", assemble(ihdr(3, 1, 2, 3), pal(4), zero_idat(3, 1, 2, 1))),
|
||||
("RGB with 6-byte tRNS and gAMA", assemble(ihdr(1, 1, 8, 2), (b"gAMA", struct.pack(">I", 45455)),
|
||||
(b"tRNS", bytes(6)), zero_idat(1, 1, 8, 3))),
|
||||
("compressed zTXt (100 KB of text) and iTXt after IDAT",
|
||||
assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"Comment\0\0" + zlib.compress(b"a" * 100_000)),
|
||||
(b"iTXt", b"Title\0\1\0ru\0\0" + zlib.compress("Привет".encode())))),
|
||||
]
|
||||
for name, data in good:
|
||||
att = upload(alice, chat["id"], data, filename="good.png").json
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
check(got.data == data, "valid PNG changed")
|
||||
print(f"{name} -> 201, downloaded byte for byte")
|
||||
print("PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted")
|
||||
|
||||
|
||||
def test_mime_substitution():
|
||||
alice, _, _, chat = setup()
|
||||
cases = [
|
||||
("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"),
|
||||
("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"),
|
||||
("GIF bytes sent as image/png", images.GIF, "image/png"),
|
||||
("PNG sent as image/gif", images.sample_png(), "image/gif"),
|
||||
("PNG sent as text/html", images.sample_png(), "text/html"),
|
||||
("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"),
|
||||
("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"),
|
||||
]
|
||||
for name, data, ctype in cases:
|
||||
check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type")
|
||||
print(f"{name} -> 415")
|
||||
att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json
|
||||
check(att["content_type"] == "image/png", "content type must be normalised")
|
||||
print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png")
|
||||
print("PASS: the type is taken from the content; a declared type that does not match is refused")
|
||||
|
||||
|
||||
def test_filenames():
|
||||
alice, _, _, chat = setup()
|
||||
bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png",
|
||||
"bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256]
|
||||
for name in bad:
|
||||
check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request")
|
||||
print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400")
|
||||
check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request")
|
||||
print("no filename -> 400")
|
||||
for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"):
|
||||
att = upload(alice, chat["id"], images.sample_png(), filename=name).json
|
||||
check(att["filename"] == name, "display name changed")
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"])
|
||||
check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded")
|
||||
print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only")
|
||||
print("PASS: filename never becomes a path; dangerous names are refused")
|
||||
|
||||
|
||||
def test_access_control():
|
||||
alice, bob, eve, chat = setup()
|
||||
att = upload(alice, chat["id"], images.sample_png()).json
|
||||
for suffix in ("", "/content"):
|
||||
reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
|
||||
check_error(reply, "not_found")
|
||||
call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
|
||||
call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
|
||||
call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
|
||||
print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400")
|
||||
check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found")
|
||||
check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found")
|
||||
upload(None, chat["id"], images.sample_png(), expected=401)
|
||||
print("upload into someone else's / unknown chat -> 404, without token -> 401")
|
||||
call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", 400, token=bob["token"])
|
||||
call("GET", f"/api/v1/attachments/{att['id']}/content?variant=original", 200, token=bob["token"])
|
||||
print("variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200")
|
||||
print("PASS: only chat members can see an attachment")
|
||||
|
||||
|
||||
def test_message_attachments():
|
||||
alice, bob, eve, chat = setup()
|
||||
first = upload(alice, chat["id"], images.sample_png()).json
|
||||
second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json
|
||||
msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json
|
||||
check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed")
|
||||
print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member")
|
||||
plain = post_message(bob, chat["id"], "без вложений").json
|
||||
check(plain["attachment_ids"] == [], "default attachment_ids must be []")
|
||||
reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json
|
||||
history, _ = read_history(bob, chat["id"])
|
||||
check(history == [msg, plain, reused], "history differs from the confirmed messages")
|
||||
print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed")
|
||||
|
||||
other_chat = new_chat(alice, eve)
|
||||
foreign = upload(alice, other_chat["id"], images.sample_png()).json
|
||||
for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"),
|
||||
([first["id"], foreign["id"]], "own + foreign attachment"),
|
||||
([str(uuid.uuid4())], "unknown attachment_id")):
|
||||
check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found")
|
||||
print(f"{what} -> 404")
|
||||
check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found")
|
||||
print("outsider referencing an attachment of the chat -> 404")
|
||||
for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"),
|
||||
([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"),
|
||||
(["not-a-uuid"], "x", "bad UUID"),
|
||||
([], "", "empty text without attachments"),
|
||||
([], " ", "whitespace text without attachments")):
|
||||
check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request")
|
||||
print(f"{what} -> 400")
|
||||
history_after, _ = read_history(bob, chat["id"])
|
||||
check(history_after == history, "a rejected message was stored")
|
||||
print("PASS: only attachments of the same chat can be linked; rejected requests change nothing")
|
||||
|
||||
|
||||
def test_idempotency_with_attachments():
|
||||
alice, bob, _, chat = setup()
|
||||
att = upload(alice, chat["id"], images.sample_png()).json
|
||||
other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json
|
||||
key = new_key()
|
||||
first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json
|
||||
again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json
|
||||
check(again == first, "replay returned a different message")
|
||||
check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict")
|
||||
check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict")
|
||||
key2 = new_key()
|
||||
plain = post_message(alice, chat["id"], "текст", None, key=key2).json
|
||||
check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field")
|
||||
history, _ = read_history(bob, chat["id"])
|
||||
check(len(history) == 2, "replays created extra messages")
|
||||
print("same key + same attachments -> same 201; other attachments -> 409; missing field == []")
|
||||
print("PASS: Idempotency-Key covers attachment_ids")
|
||||
|
||||
|
||||
def test_aborted_upload():
|
||||
alice, _, _, chat = setup()
|
||||
sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10)
|
||||
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n"
|
||||
f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n")
|
||||
sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000))
|
||||
sock.close() # клиент пропал на середине тела
|
||||
print("sent 1.1 MB of a declared 5 MB body and closed the connection")
|
||||
att = upload(alice, chat["id"], images.sample_png()).json
|
||||
call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"])
|
||||
print("the API keeps serving uploads; nothing of the aborted request is visible "
|
||||
"(DB/S3 side is checked by test_consistency.py)")
|
||||
print("PASS: an interrupted upload is dropped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming,
|
||||
test_pixel_limit, test_damaged_images, test_png_chunk_semantics, test_mime_substitution, test_filenames, test_access_control,
|
||||
test_message_attachments, test_idempotency_with_attachments, test_aborted_upload])
|
||||
@@ -0,0 +1,103 @@
|
||||
from collections import Counter
|
||||
|
||||
import images
|
||||
from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload
|
||||
|
||||
|
||||
def test_requests_spread_over_replicas():
|
||||
alice = new_user("alice")
|
||||
replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)]
|
||||
served = Counter(reply.instance for reply in replies)
|
||||
print(f"20 x GET /users/me with one token -> replicas: {dict(served)}")
|
||||
check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned")
|
||||
check(len(served) >= 2, "all requests went to one replica: balancing does not work")
|
||||
print("PASS: one session works on every replica, no second login (sessions are shared in Redis)")
|
||||
|
||||
|
||||
def test_write_on_one_read_on_other():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
sent = send(alice, chat["id"], "written on one replica")
|
||||
readers = set()
|
||||
for _ in range(6):
|
||||
reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"])
|
||||
check(sent.json in reply.json["items"], "confirmed message is not visible on another replica")
|
||||
readers.add(reply.instance)
|
||||
print(f"message written by {sent.instance}; read back by {sorted(readers)}")
|
||||
check(readers - {sent.instance}, "no read went to another replica")
|
||||
print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)")
|
||||
|
||||
|
||||
def test_logout_seen_by_all_replicas():
|
||||
user = new_user("logout")
|
||||
out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"])
|
||||
after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)]
|
||||
print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}")
|
||||
check(len(set(after)) >= 2, "only one replica answered")
|
||||
print("PASS: logout on one replica revokes the session on all of them")
|
||||
|
||||
|
||||
def test_idempotent_retry_on_other_replica():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
key = new_key()
|
||||
first = send(alice, chat["id"], "retry me", key=key)
|
||||
for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику
|
||||
retry = send(alice, chat["id"], "retry me", key=key)
|
||||
if retry.instance != first.instance:
|
||||
break
|
||||
print(f"first POST on {first.instance} -> 201 {first.json['id']}; "
|
||||
f"same key on {retry.instance} -> {retry.status} {retry.json['id']}")
|
||||
check(retry.instance != first.instance, "retry never reached another replica")
|
||||
check(retry.json == first.json, "retry on another replica returned a different message")
|
||||
items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"]
|
||||
check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate")
|
||||
print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message")
|
||||
|
||||
|
||||
def test_attachment_on_other_replica():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
data = images.sample_png()
|
||||
sent = upload(alice, chat["id"], data)
|
||||
readers = set()
|
||||
for _ in range(6):
|
||||
reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"])
|
||||
check(reply.data == data, "another replica returned different bytes")
|
||||
readers.add(reply.instance)
|
||||
print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}")
|
||||
check(readers - {sent.instance}, "no download went to another replica")
|
||||
print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)")
|
||||
|
||||
|
||||
def test_instance_header_everywhere():
|
||||
containers = api_containers()
|
||||
alice, eve = new_user("alice"), new_user("eve")
|
||||
chat = new_chat(alice)
|
||||
replies = [
|
||||
call("GET", "/health/live", 200),
|
||||
call("GET", "/health/ready", 200),
|
||||
send(alice, chat["id"], "hello"),
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}),
|
||||
call("GET", "/api/v1/users/me", 401),
|
||||
call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]),
|
||||
call("POST", "/api/v1/users", 409,
|
||||
body={"username": alice["username"], "display_name": "x", "password": alice["password"]}),
|
||||
]
|
||||
for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер
|
||||
check(any(container.startswith(reply.instance) for container in containers),
|
||||
f"X-Instance-Id {reply.instance} is not a running api container")
|
||||
print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container "
|
||||
f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}")
|
||||
print("PASS: X-Instance-Id is in every response and points to a real container")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([
|
||||
test_requests_spread_over_replicas,
|
||||
test_write_on_one_read_on_other,
|
||||
test_logout_seen_by_all_replicas,
|
||||
test_idempotent_retry_on_other_replica,
|
||||
test_attachment_on_other_replica,
|
||||
test_instance_header_everywhere,
|
||||
])
|
||||
@@ -0,0 +1,275 @@
|
||||
"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка).
|
||||
|
||||
Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`),
|
||||
выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает
|
||||
janitor вручную. Данные синтетические, volumes не удаляются.
|
||||
|
||||
python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]
|
||||
"""
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
import images
|
||||
from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat,
|
||||
new_user, post_message, run_tests, sql_value, upload, wait_ready)
|
||||
|
||||
URL = urllib.parse.urlsplit(BASE_URL)
|
||||
|
||||
# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но finish_upload
|
||||
# (UPDATE ... state='ready' + COMMIT) падает — как если бы реплика умерла сразу после PUT в S3.
|
||||
CRASH_AFTER_PUT = r'''
|
||||
import json, struct, sys, tempfile, zlib
|
||||
from uuid import UUID
|
||||
import attachments, db, storage, uploads
|
||||
chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2])
|
||||
def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d))
|
||||
data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b""))
|
||||
f = tempfile.SpooledTemporaryFile(); f.write(data)
|
||||
import hashlib
|
||||
upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest())
|
||||
image = uploads.inspect(upload, "image/png")
|
||||
created = {}
|
||||
real_intent = db.create_upload_intent
|
||||
def remember(attachment_id, *args):
|
||||
created["id"], created["key"] = str(attachment_id), args[-1]
|
||||
return real_intent(attachment_id, *args)
|
||||
def crash(attachment_id):
|
||||
raise RuntimeError("simulated crash between S3 PUT and DB COMMIT")
|
||||
db.create_upload_intent = remember
|
||||
db.finish_upload = crash
|
||||
db.pool.open()
|
||||
try:
|
||||
attachments.store(chat_id, owner_id, "crash.png", upload, image)
|
||||
outcome = "stored"
|
||||
except RuntimeError as exc:
|
||||
outcome = str(exc)
|
||||
print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"],
|
||||
"object_exists": storage.exists(created["key"])}))
|
||||
'''
|
||||
|
||||
OBJECT_EXISTS = r'''
|
||||
import json, sys, storage
|
||||
print(json.dumps({"exists": storage.exists(sys.argv[1])}))
|
||||
'''
|
||||
|
||||
PUT_ORPHAN = r'''
|
||||
import hashlib, io, json, uuid, storage
|
||||
data = b"orphan object without a row in PostgreSQL"
|
||||
key = storage.new_key(uuid.uuid4().hex)
|
||||
storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest())
|
||||
print(json.dumps({"key": key}))
|
||||
'''
|
||||
|
||||
DELETE_OBJECT = r'''
|
||||
import json, sys, storage
|
||||
storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]}))
|
||||
'''
|
||||
|
||||
COUNT_OBJECTS = r'''
|
||||
import json, storage
|
||||
print(json.dumps({"objects": sum(1 for _ in storage.list_objects())}))
|
||||
'''
|
||||
|
||||
|
||||
def last_json(output):
|
||||
return json.loads(output.splitlines()[-1])
|
||||
|
||||
|
||||
def object_exists(key):
|
||||
return last_json(app_python(OBJECT_EXISTS, key))["exists"]
|
||||
|
||||
|
||||
def backdate(where):
|
||||
sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}")
|
||||
|
||||
|
||||
def uploading_rows(chat_id):
|
||||
return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'"))
|
||||
|
||||
|
||||
def wait_download(user, att, seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"])
|
||||
if reply.status == 200:
|
||||
return reply
|
||||
check(time.monotonic() < deadline, "download did not recover after S3 start")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def s3_outage(stop_action, start_action, label):
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
data = images.sample_png()
|
||||
att = upload(alice, chat["id"], data).json
|
||||
before = uploading_rows(chat["id"])
|
||||
print(f"--- S3 {label}: docker compose {stop_action} s3")
|
||||
compose(stop_action, "s3")
|
||||
try:
|
||||
started = time.monotonic()
|
||||
reply = upload(alice, chat["id"], data, expected=503)
|
||||
ms = (time.monotonic() - started) * 1000
|
||||
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
|
||||
body = reply.data.decode().lower()
|
||||
check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")),
|
||||
"error leaks storage details")
|
||||
print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body")
|
||||
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
|
||||
print("download of an existing file -> 503")
|
||||
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json
|
||||
check(meta["sha256"] == att["sha256"], "metadata changed")
|
||||
post_message(alice, chat["id"], "текст без файла")
|
||||
post_message(alice, chat["id"], "", [att["id"]])
|
||||
call("GET", "/health/ready", 200)
|
||||
print("metadata, messages (also with an already stored attachment) and /health/ready keep working: "
|
||||
"S3 is needed only for file bytes")
|
||||
after = uploading_rows(chat["id"])
|
||||
print(f"uploading rows of this chat: before {before}, after the failed upload {after} "
|
||||
"(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API")
|
||||
finally:
|
||||
print(f"docker compose {start_action} s3")
|
||||
compose(start_action, "s3")
|
||||
got = wait_download(bob, att)
|
||||
check(got.data == data, "file changed after S3 recovery")
|
||||
print("after S3 recovery the old file downloads byte for byte")
|
||||
upload(alice, chat["id"], data)
|
||||
print("new upload -> 201")
|
||||
leftover = uploading_rows(chat["id"])
|
||||
if leftover:
|
||||
backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'")
|
||||
report = janitor()
|
||||
print(f"janitor after grace period: {json.dumps(report)}")
|
||||
check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain")
|
||||
print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up")
|
||||
|
||||
|
||||
def test_s3_stopped():
|
||||
s3_outage("stop", "start", "stopped (connection refused / name not resolved)")
|
||||
|
||||
|
||||
def test_s3_frozen():
|
||||
s3_outage("pause", "unpause", "frozen (no answer, timeouts)")
|
||||
|
||||
|
||||
def test_crash_between_s3_and_db():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
ready = upload(alice, chat["id"], images.sample_png()).json
|
||||
result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"]))
|
||||
check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}")
|
||||
att_id, key = result["attachment_id"], result["object_key"]
|
||||
state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'")
|
||||
check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3")
|
||||
print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3")
|
||||
call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"])
|
||||
call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"])
|
||||
post_message(alice, chat["id"], "x", [att_id], expected=404)
|
||||
print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404")
|
||||
|
||||
report = janitor()
|
||||
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key),
|
||||
"fresh upload was deleted inside the grace period")
|
||||
print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> "
|
||||
"fresh row and object are kept (the upload might still be running)")
|
||||
backdate(f"id = '{att_id}'")
|
||||
report = janitor("--dry-run")
|
||||
check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report")
|
||||
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row")
|
||||
print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted")
|
||||
report = janitor()
|
||||
check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload")
|
||||
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists")
|
||||
check(not object_exists(key), "object still exists")
|
||||
print(f"janitor: {json.dumps(report)} -> row and object deleted")
|
||||
report = janitor()
|
||||
print(f"janitor again: {json.dumps(report)} -> idempotent")
|
||||
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
|
||||
check(got.data == images.sample_png(), "ready attachment damaged by cleanup")
|
||||
print("the ready attachment of the same chat is untouched")
|
||||
print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period")
|
||||
|
||||
|
||||
def test_orphan_objects():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
ready = upload(alice, chat["id"], images.sample_png()).json
|
||||
key = last_json(app_python(PUT_ORPHAN))["key"]
|
||||
print(f"object {key} put into the bucket without a row in PostgreSQL")
|
||||
# Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с.
|
||||
short = {"UPLOAD_TIMEOUT_SECONDS": "1"}
|
||||
report = janitor("--grace-seconds", "3", env=short)
|
||||
check(object_exists(key), "fresh orphan deleted inside grace period")
|
||||
print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept")
|
||||
time.sleep(4)
|
||||
report = janitor("--grace-seconds", "3", "--dry-run", env=short)
|
||||
check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key),
|
||||
"dry run must report and keep the orphan")
|
||||
print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept")
|
||||
report = janitor("--grace-seconds", "3", env=short)
|
||||
check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted")
|
||||
print(f"janitor: {json.dumps(report)} -> orphan deleted")
|
||||
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
|
||||
check(got.data == images.sample_png(), "ready attachment was deleted with orphans")
|
||||
print("ready attachments older than the grace period are untouched (they have rows)")
|
||||
|
||||
victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json
|
||||
victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'")
|
||||
app_python(DELETE_OBJECT, victim_key)
|
||||
report = janitor("--grace-seconds", "3", env=short)
|
||||
check(report["missing_objects"] >= 1, "missing object not reported")
|
||||
check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted")
|
||||
reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"])
|
||||
print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects="
|
||||
f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)")
|
||||
print("PASS: orphans are found and deleted only after the grace period; missing objects are reported")
|
||||
|
||||
|
||||
def api_tmp_files():
|
||||
counts = {}
|
||||
for container in compose_output("ps", "-q", "api").split():
|
||||
counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0)
|
||||
return counts
|
||||
|
||||
|
||||
def test_aborted_upload_leaves_nothing():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
objects = last_json(app_python(COUNT_OBJECTS))["objects"]
|
||||
rows = int(sql_value("SELECT count(*) FROM attachments"))
|
||||
for sent in (100_000, 3_000_000):
|
||||
sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10)
|
||||
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n"
|
||||
f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n"
|
||||
f"Content-Length: 9000000\r\n\r\n")
|
||||
sock.sendall(head.encode() + images.sample_png() + bytes(sent))
|
||||
sock.close()
|
||||
print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection")
|
||||
time.sleep(2)
|
||||
check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row")
|
||||
check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object")
|
||||
print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}")
|
||||
tmp = api_tmp_files()
|
||||
check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}")
|
||||
print(f"temporary upload files in api containers: {tmp} (removed when the request ends)")
|
||||
for container in compose_output("ps", "-q", "api").split():
|
||||
mounts = docker("inspect", "-f", "{{json .Mounts}}", container)
|
||||
# файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных
|
||||
storage = [m for m in json.loads(mounts)
|
||||
if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)]
|
||||
check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}")
|
||||
print("api containers have no volumes: the original is never stored locally")
|
||||
print("PASS: an interrupted upload leaves no row, no object and no local file")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
target = sys.argv[1] if len(sys.argv) > 1 else "all"
|
||||
groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db],
|
||||
"orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]}
|
||||
groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"]
|
||||
check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]")
|
||||
wait_ready()
|
||||
run_tests(groups[target])
|
||||
@@ -0,0 +1,101 @@
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from common import call, check, compose_output, new_user, psql, run_tests
|
||||
|
||||
|
||||
def sql_value(sql):
|
||||
code, output, error = psql(sql)
|
||||
check(code == 0, f"psql failed: {error}")
|
||||
return output
|
||||
|
||||
|
||||
def test_create_chat_rolls_back():
|
||||
alice = new_user("rollback")
|
||||
title = "rollback-" + uuid.uuid4().hex[:8]
|
||||
inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
|
||||
|
||||
call("POST", "/api/v1/chats", 400, token=alice["token"],
|
||||
body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]})
|
||||
|
||||
left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'")
|
||||
orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS "
|
||||
"(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)")
|
||||
inserted_after = inserted_before
|
||||
deadline = time.monotonic() + 15
|
||||
while inserted_after == inserted_before and time.monotonic() < deadline:
|
||||
time.sleep(0.5)
|
||||
inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
|
||||
print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; "
|
||||
f"chats insert counter: {inserted_before} -> {inserted_after}")
|
||||
check(left == "0" and orphans == "0", "chat row survived the failed transaction")
|
||||
check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown")
|
||||
print("PASS: chat row was inserted and then rolled back together with members")
|
||||
|
||||
|
||||
def test_message_needs_membership_in_db():
|
||||
code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) "
|
||||
f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))")
|
||||
print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}")
|
||||
check(code != 0 and "foreign key" in error, "database accepted a message from a non-member")
|
||||
print("PASS: the database itself rejects a message from a non-member")
|
||||
|
||||
|
||||
def test_password_is_argon2id():
|
||||
user = new_user("hash")
|
||||
stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'")
|
||||
print(f"stored password: {stored[:31]}... (length {len(stored)})")
|
||||
check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash")
|
||||
print("PASS: password is stored as Argon2id hash")
|
||||
|
||||
|
||||
def test_unique_username_constraint():
|
||||
constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint "
|
||||
"WHERE conrelid = 'users'::regclass AND contype = 'u'")
|
||||
print(f"unique constraint on users: {constraint}")
|
||||
check(constraint == "UNIQUE (username)", "no unique constraint on username")
|
||||
print("PASS: username is unique in the database")
|
||||
|
||||
|
||||
def test_migrations_are_not_repeated():
|
||||
versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations")
|
||||
print(f"applied migrations: {versions}; running the migrate service once more")
|
||||
output = compose_output("run", "--rm", "--no-deps", "-T", "migrate")
|
||||
again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations")
|
||||
last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1]
|
||||
print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}")
|
||||
check(again == versions and "none" in last_log, "migrations were applied again")
|
||||
print("PASS: migrations are applied once, a repeated run does nothing")
|
||||
|
||||
|
||||
def test_replicas_do_not_migrate():
|
||||
"""lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему."""
|
||||
logs = compose_output("logs", "api").splitlines()
|
||||
checked = [line for line in logs if "Schema is up to date" in line]
|
||||
migrated = [line for line in logs if "Migrations applied" in line]
|
||||
migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip()
|
||||
print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': "
|
||||
f"{len(migrated)}; migrate service: {migrate_exit}")
|
||||
check(checked and not migrated, "api replicas must not apply migrations")
|
||||
check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0")
|
||||
print("PASS: only the migrate service applies migrations, replicas wait for the schema")
|
||||
|
||||
|
||||
def test_idempotency_key_is_unique_in_db():
|
||||
constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint "
|
||||
"WHERE conname = 'messages_idempotency_key'")
|
||||
print(f"unique constraint for Idempotency-Key: {constraint}")
|
||||
check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys")
|
||||
print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([
|
||||
test_create_chat_rolls_back,
|
||||
test_message_needs_membership_in_db,
|
||||
test_password_is_argon2id,
|
||||
test_unique_username_constraint,
|
||||
test_migrations_are_not_repeated,
|
||||
test_replicas_do_not_migrate,
|
||||
test_idempotency_key_is_unique_in_db,
|
||||
])
|
||||
@@ -0,0 +1,57 @@
|
||||
import datetime as dt
|
||||
import os
|
||||
import time
|
||||
|
||||
from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready
|
||||
|
||||
SHORT_TTL = 2
|
||||
|
||||
|
||||
def restart_api(ttl=None):
|
||||
env = dict(os.environ)
|
||||
if ttl is None:
|
||||
env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env
|
||||
else:
|
||||
env["SESSION_TTL_SECONDS"] = str(ttl)
|
||||
compose("up", "-d", "--no-deps", "--wait", "api", env=env)
|
||||
wait_ready()
|
||||
|
||||
|
||||
def seconds_left(expires_at):
|
||||
moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00"))
|
||||
return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds()
|
||||
|
||||
|
||||
def test_session_expires():
|
||||
print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}")
|
||||
restart_api(SHORT_TTL)
|
||||
try:
|
||||
user = new_user("ttl")
|
||||
session = login(user["username"], user["password"]).json
|
||||
token = session["token"]
|
||||
left = seconds_left(session["expires_at"])
|
||||
print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: "
|
||||
f"{redis_cli('pttl', session_key(token))} ms")
|
||||
check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions")
|
||||
|
||||
call("GET", "/api/v1/users/me", 200, token=token)
|
||||
print("before expiry: GET /users/me -> 200")
|
||||
|
||||
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0)
|
||||
call("GET", "/api/v1/users/me", 401, token=token)
|
||||
exists = redis_cli("exists", session_key(token))
|
||||
print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}")
|
||||
check(exists == "0", "Redis must delete the expired session key")
|
||||
finally:
|
||||
print("recreating api with SESSION_TTL_SECONDS from .env")
|
||||
restart_api()
|
||||
|
||||
user = new_user("ttl_back")
|
||||
left = seconds_left(login(user["username"], user["password"]).json["expires_at"])
|
||||
print(f"normal API again: new session expires in {left:.0f} s")
|
||||
check(left > 300, "normal TTL was not restored")
|
||||
print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([test_session_expires])
|
||||
@@ -0,0 +1,110 @@
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready
|
||||
|
||||
FAKE_TOKEN = "A" * 43
|
||||
|
||||
|
||||
def timed(method, path, expected, **kwargs):
|
||||
started = time.monotonic()
|
||||
reply = call(method, path, expected, **kwargs)
|
||||
return reply, (time.monotonic() - started) * 1000
|
||||
|
||||
|
||||
def prepare():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
before = send(alice, chat["id"], "confirmed before outage").json
|
||||
return alice, bob, chat, before
|
||||
|
||||
|
||||
def recovered(alice, chat, before):
|
||||
wait_ready()
|
||||
history, _ = read_history(alice, chat["id"])
|
||||
check(before in history, "message confirmed before the outage is lost or changed")
|
||||
call("GET", "/api/v1/users/me", 200, token=alice["token"])
|
||||
print("after start: the old token works, confirmed messages are in place")
|
||||
|
||||
|
||||
def test_redis_down():
|
||||
alice, bob, chat, before = prepare()
|
||||
print("stopping redis")
|
||||
compose("stop", "redis")
|
||||
try:
|
||||
call("GET", "/health/live", 200)
|
||||
_, ms = timed("GET", "/health/ready", 503)
|
||||
print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)")
|
||||
reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"])
|
||||
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
|
||||
print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked")
|
||||
call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN)
|
||||
print("unknown token -> 503, not 200: nobody is trusted without a check")
|
||||
call("GET", "/api/v1/chats", 401)
|
||||
print("no token -> 401")
|
||||
call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)})
|
||||
print("login -> 503: there is nowhere to store the session")
|
||||
call("POST", "/api/v1/users", 201,
|
||||
body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD})
|
||||
print("registration -> 201: it needs only PostgreSQL")
|
||||
finally:
|
||||
print("starting redis")
|
||||
compose("start", "redis")
|
||||
recovered(alice, chat, before)
|
||||
print("PASS: Redis outage gives 503 and does not let anyone in without a session check")
|
||||
|
||||
|
||||
def test_redis_crash():
|
||||
alice = new_user("crash")
|
||||
print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown")
|
||||
compose("kill", "redis")
|
||||
try:
|
||||
# Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy.
|
||||
status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status
|
||||
print(f"right after the crash: GET /users/me -> {status}")
|
||||
finally:
|
||||
print("starting redis")
|
||||
compose("start", "redis")
|
||||
wait_ready()
|
||||
call("GET", "/api/v1/users/me", 200, token=alice["token"])
|
||||
print("after start: the session issued before the crash still works (AOF, appendfsync always)")
|
||||
print("PASS: sessions survive a Redis crash, not only a clean restart")
|
||||
|
||||
|
||||
def test_postgres_down():
|
||||
alice, bob, chat, before = prepare()
|
||||
print("stopping postgres")
|
||||
compose("stop", "postgres")
|
||||
try:
|
||||
call("GET", "/health/live", 200)
|
||||
_, ms = timed("GET", "/health/ready", 503)
|
||||
print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)")
|
||||
for method, path, body in (("GET", "/api/v1/users/me", None),
|
||||
("GET", "/api/v1/chats", None),
|
||||
("GET", f"/api/v1/chats/{chat['id']}/messages", None),
|
||||
("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})):
|
||||
reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()})
|
||||
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
|
||||
check("postgres" not in reply.data.decode().lower(), "error must not show database details")
|
||||
print("valid token: users/me, chats, history, new message -> 503 unavailable, no details")
|
||||
call("GET", "/api/v1/chats", 401)
|
||||
call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN)
|
||||
print("no token -> 401, unknown token -> 401: anonymous users are not let in")
|
||||
call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)})
|
||||
call("POST", "/api/v1/users", 503,
|
||||
body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD})
|
||||
print("login -> 503, registration -> 503")
|
||||
finally:
|
||||
print("starting postgres")
|
||||
compose("start", "postgres")
|
||||
recovered(alice, chat, before)
|
||||
print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
target = sys.argv[1] if len(sys.argv) > 1 else "all"
|
||||
tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down],
|
||||
"all": [test_redis_down, test_redis_crash, test_postgres_down]}
|
||||
check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]")
|
||||
run_tests(tests[target])
|
||||
@@ -0,0 +1,174 @@
|
||||
import json
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send
|
||||
|
||||
PARALLEL = 20
|
||||
|
||||
|
||||
def post_raw(user, chat_id, raw, key):
|
||||
"""POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы."""
|
||||
return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"],
|
||||
raw=raw, headers={"Idempotency-Key": key})
|
||||
|
||||
|
||||
def texts_in(user, chat_id):
|
||||
history, _ = read_history(user, chat_id)
|
||||
return [message["text"] for message in history]
|
||||
|
||||
|
||||
def test_replay_returns_same_message():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
key = new_key()
|
||||
first = send(alice, chat["id"], "Привет 👋", key=key)
|
||||
# То же JSON-значение, записанное иначе: пробелы и \u-экранирование.
|
||||
same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key)
|
||||
again = send(alice, chat["id"], "Привет 👋", key=key)
|
||||
print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}")
|
||||
check(same.status == 201 and same.json == first.json and again.json == first.json,
|
||||
"retry must return the original 201 and the same message")
|
||||
check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate")
|
||||
print("PASS: a retry with the same key returns the original 201 and message, no duplicate")
|
||||
|
||||
|
||||
def test_same_key_parallel_through_replicas():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
key = new_key()
|
||||
barrier = threading.Barrier(PARALLEL)
|
||||
|
||||
def post(_):
|
||||
barrier.wait() # все запросы стартуют одновременно
|
||||
return send(alice, chat["id"], "one message for 20 requests", key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
|
||||
replies = list(pool.map(post, range(PARALLEL)))
|
||||
bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies}
|
||||
served = Counter(reply.instance for reply in replies)
|
||||
count = texts_in(bob, chat["id"]).count("one message for 20 requests")
|
||||
print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, "
|
||||
f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}")
|
||||
check(len(served) >= 2, "requests did not go through different replicas")
|
||||
check(len(bodies) == 1 and count == 1, "parallel retries created more than one message")
|
||||
print("PASS: parallel requests with one key through different replicas create exactly one message")
|
||||
|
||||
|
||||
def test_same_key_different_bodies_parallel():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
key = new_key()
|
||||
barrier = threading.Barrier(PARALLEL)
|
||||
|
||||
def post(index):
|
||||
text = "body A" if index % 2 == 0 else "body B"
|
||||
barrier.wait()
|
||||
reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"],
|
||||
body={"text": text}, headers={"Idempotency-Key": key})
|
||||
return text, reply
|
||||
|
||||
with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
|
||||
results = list(pool.map(post, range(PARALLEL)))
|
||||
texts = texts_in(alice, chat["id"])
|
||||
winner = [t for t in texts if t in ("body A", "body B")]
|
||||
print(f"{PARALLEL} parallel POSTs, one key, two different bodies: "
|
||||
f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}")
|
||||
check(len(winner) == 1, "exactly one message must be created")
|
||||
for text, reply in results:
|
||||
expected = 201 if text == winner[0] else 409
|
||||
check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}")
|
||||
print("PASS: the first body wins, the same body gets 201, the other body gets 409")
|
||||
|
||||
|
||||
def test_key_scope():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
other = new_chat(alice, bob)
|
||||
key = new_key()
|
||||
a1 = send(alice, chat["id"], "scope", key=key).json
|
||||
b1 = send(bob, chat["id"], "scope", key=key).json
|
||||
a2 = send(alice, other["id"], "scope", key=key).json
|
||||
print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages")
|
||||
check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat")
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"],
|
||||
body={"text": "other text"}, headers={"Idempotency-Key": key})
|
||||
print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409")
|
||||
|
||||
|
||||
def test_key_validation():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
call("POST", path, 400, token=alice["token"], body={"text": "no key"})
|
||||
for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"):
|
||||
call("POST", path, 400, token=alice["token"], body={"text": "bad key"},
|
||||
headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8
|
||||
longest = ("Az09._:-" * 16)[:128]
|
||||
call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest})
|
||||
print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201")
|
||||
print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
def test_failed_request_does_not_burn_key():
|
||||
alice, eve = new_user("alice"), new_user("eve")
|
||||
chat = new_chat(alice)
|
||||
key = new_key()
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key})
|
||||
call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key})
|
||||
created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key})
|
||||
print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}")
|
||||
print("PASS: rejected requests (400, 404) do not store a result for the key")
|
||||
|
||||
|
||||
def test_replay_checks_session_and_access():
|
||||
alice, eve = new_user("alice"), new_user("eve")
|
||||
chat = new_chat(alice)
|
||||
key = new_key()
|
||||
send(alice, chat["id"], "private", key=key)
|
||||
stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"],
|
||||
body={"text": "private"}, headers={"Idempotency-Key": key})
|
||||
call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
||||
revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"],
|
||||
body={"text": "private"}, headers={"Idempotency-Key": key})
|
||||
print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}")
|
||||
print("PASS: a retry checks the session and chat access again, the saved answer is not leaked")
|
||||
|
||||
|
||||
def test_lost_response_then_retry():
|
||||
"""Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен."""
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
key = new_key()
|
||||
body = json.dumps({"text": "answer was lost"}).encode("utf-8")
|
||||
url = urllib.parse.urlsplit(BASE_URL)
|
||||
request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n"
|
||||
f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n"
|
||||
f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode()
|
||||
with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection:
|
||||
connection.sendall(request + body) # и сразу закрываем, ответ не читаем
|
||||
time.sleep(1)
|
||||
retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"],
|
||||
body={"text": "answer was lost"}, headers={"Idempotency-Key": key})
|
||||
count = texts_in(alice, chat["id"]).count("answer was lost")
|
||||
print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}")
|
||||
check(count == 1, "the retry after a lost answer created a duplicate")
|
||||
print("PASS: an unconfirmed POST can be safely retried with the same key")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([
|
||||
test_replay_returns_same_message,
|
||||
test_same_key_parallel_through_replicas,
|
||||
test_same_key_different_bodies_parallel,
|
||||
test_key_scope,
|
||||
test_key_validation,
|
||||
test_failed_request_does_not_burn_key,
|
||||
test_replay_checks_session_and_access,
|
||||
test_lost_response_then_retry,
|
||||
])
|
||||
@@ -0,0 +1,57 @@
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send
|
||||
|
||||
CLIENTS = 10
|
||||
MESSAGES_PER_CLIENT = 10
|
||||
REGISTRATIONS = 20
|
||||
|
||||
|
||||
def test_concurrent_messages():
|
||||
users = [new_user(f"race{i}") for i in range(CLIENTS)]
|
||||
chat = new_chat(*users, title="race")
|
||||
barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно
|
||||
|
||||
def client(index):
|
||||
barrier.wait()
|
||||
for n in range(MESSAGES_PER_CLIENT):
|
||||
send(users[index], chat["id"], f"client-{index}-message-{n}")
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
|
||||
list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
history, pages = read_history(users[0], chat["id"], limit=30)
|
||||
ids = {message["id"] for message in history}
|
||||
expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)}
|
||||
print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; "
|
||||
f"read {len(history)} in {pages} pages, unique ids: {len(ids)}")
|
||||
check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages")
|
||||
check(len(ids) == len(history), "duplicate message ids")
|
||||
check({message["text"] for message in history} == expected, "lost or changed message texts")
|
||||
print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates")
|
||||
|
||||
|
||||
def test_concurrent_registration():
|
||||
username = "race_" + uuid.uuid4().hex[:10]
|
||||
barrier = threading.Barrier(REGISTRATIONS)
|
||||
|
||||
def register(index):
|
||||
barrier.wait()
|
||||
body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD}
|
||||
return call("POST", "/api/v1/users", (201, 409), body=body).status
|
||||
|
||||
with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool:
|
||||
statuses = list(pool.map(register, range(REGISTRATIONS)))
|
||||
created, conflicts = statuses.count(201), statuses.count(409)
|
||||
print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}")
|
||||
check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win")
|
||||
print("PASS: concurrent registration created exactly one user")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([test_concurrent_messages, test_concurrent_registration])
|
||||
@@ -0,0 +1,146 @@
|
||||
"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен.
|
||||
|
||||
Нужен запущенный стенд из compose.yaml.
|
||||
"""
|
||||
import json
|
||||
import socket
|
||||
|
||||
import images
|
||||
from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload
|
||||
|
||||
# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API.
|
||||
ANONYMOUS = r'''
|
||||
import json, sys, urllib.request, urllib.error
|
||||
import settings
|
||||
base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET
|
||||
def status(method, url, data=None):
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r:
|
||||
return r.status, r.read(300).decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read(300).decode("utf-8", "replace")
|
||||
out = {}
|
||||
for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None),
|
||||
("head_object", "HEAD", base + "/" + sys.argv[1], None),
|
||||
("list_bucket", "GET", base + "?list-type=2", None),
|
||||
("put_object", "PUT", base + "/originals/anonymous-test", b"x"),
|
||||
("get_bucket_policy", "GET", base + "?policy", None)):
|
||||
code, body = status(method, url, data)
|
||||
out[name] = {"status": code, "s3_code": body.split("<Code>")[1].split("</Code>")[0] if "<Code>" in body else ""}
|
||||
print(json.dumps(out))
|
||||
'''
|
||||
|
||||
# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json).
|
||||
API_KEY_RIGHTS = r'''
|
||||
import io, json, sys, uuid
|
||||
from botocore.exceptions import ClientError
|
||||
import settings, storage
|
||||
c, bucket = storage._client, settings.S3_BUCKET
|
||||
def attempt(name, fn):
|
||||
try:
|
||||
fn(); return "allowed"
|
||||
except ClientError as e:
|
||||
return "denied: " + e.response.get("Error", {}).get("Code", "?")
|
||||
public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]},
|
||||
"Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]})
|
||||
key = "originals/" + uuid.uuid4().hex
|
||||
result = {
|
||||
"get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()),
|
||||
"put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")),
|
||||
"delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)),
|
||||
"list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)),
|
||||
"create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])),
|
||||
"make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)),
|
||||
"read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)),
|
||||
"delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)),
|
||||
"change versioning": attempt("versioning", lambda: c.put_bucket_versioning(
|
||||
Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})),
|
||||
}
|
||||
try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket
|
||||
result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", []))
|
||||
except ClientError as e:
|
||||
result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?")
|
||||
print(json.dumps(result))
|
||||
'''
|
||||
|
||||
|
||||
def last_json(output):
|
||||
return json.loads(output.splitlines()[-1])
|
||||
|
||||
|
||||
def prepare():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
att = upload(alice, chat["id"], images.sample_png()).json
|
||||
key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'")
|
||||
return alice, bob, att, key
|
||||
|
||||
|
||||
def test_bucket_is_private():
|
||||
alice, bob, att, key = prepare()
|
||||
result = last_json(app_python(ANONYMOUS, key))
|
||||
for name, reply in result.items():
|
||||
print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}")
|
||||
check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}")
|
||||
call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
print("the same file through the API for a chat member -> 200")
|
||||
print("PASS: the bucket is private: no anonymous read, list, write or policy read")
|
||||
|
||||
|
||||
def test_api_key_is_limited():
|
||||
alice, bob, att, key = prepare()
|
||||
result = last_json(app_python(API_KEY_RIGHTS, key))
|
||||
allowed = {"get own object", "put object", "delete object", "list bucket"}
|
||||
visible = result.pop("visible buckets")
|
||||
print(f"API key: {'list buckets':<32} -> {visible}")
|
||||
check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets")
|
||||
for name, outcome in result.items():
|
||||
print(f"API key: {name:<32} -> {outcome}")
|
||||
if name in allowed:
|
||||
check(outcome == "allowed", f"API key must be allowed to {name}")
|
||||
else:
|
||||
check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}")
|
||||
print("PASS: the API key can only get/put/delete/list objects of its own bucket; "
|
||||
"bucket admin, policy and other buckets are denied")
|
||||
|
||||
|
||||
def test_no_open_ports_or_console():
|
||||
config = json.loads(compose_output("config", "--format", "json"))
|
||||
s3 = config["services"]["s3"]
|
||||
check(not s3.get("ports"), "s3 must not publish ports")
|
||||
check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network")
|
||||
check(config["networks"]["data"].get("internal") is True, "data network must be internal")
|
||||
print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, "
|
||||
"filer UI 8888 are reachable only inside the stand)")
|
||||
for port in (8333, 9333, 8888, 19333):
|
||||
sock = socket.socket()
|
||||
sock.settimeout(1)
|
||||
refused = sock.connect_ex(("127.0.0.1", port)) != 0
|
||||
sock.close()
|
||||
check(refused, f"127.0.0.1:{port} is reachable from the host")
|
||||
print(f"127.0.0.1:{port} from the host -> connection refused")
|
||||
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()}
|
||||
for name, items in sorted(secrets.items()):
|
||||
print(f"secrets of {name}: {items or '-'}")
|
||||
for name in ("api", "janitor"):
|
||||
check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"},
|
||||
f"{name} must not get S3 admin keys or the key config")
|
||||
check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key")
|
||||
print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init")
|
||||
|
||||
|
||||
def test_bootstrap_is_idempotent():
|
||||
alice, bob, att, key = prepare()
|
||||
output = compose_output("run", "--rm", "--no-deps", "s3-init")
|
||||
print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip()))
|
||||
check("bucket is private" in output, "bootstrap did not confirm a private bucket")
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
check(got.data == images.sample_png(), "file changed after repeated bootstrap")
|
||||
upload(alice, att["chat_id"], images.sample_png())
|
||||
print("after the repeated bootstrap: old file downloads, new upload works")
|
||||
print("PASS: bucket/user/policy bootstrap can be repeated safely")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console,
|
||||
test_bootstrap_is_idempotent])
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик.
|
||||
|
||||
Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что
|
||||
тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково
|
||||
(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше
|
||||
DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование)
|
||||
при тех же 10 загрузках не помещается в лимит 512 MiB.
|
||||
|
||||
Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков
|
||||
одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status
|
||||
(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера
|
||||
(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с.
|
||||
|
||||
python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json]
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
import statistics
|
||||
import threading
|
||||
import time
|
||||
|
||||
import images
|
||||
from common import ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_balanced, wait_ready
|
||||
|
||||
|
||||
def proc_status(container):
|
||||
"""VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты)."""
|
||||
script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; "
|
||||
"for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current "
|
||||
"/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; "
|
||||
"do [ -r $f ] && echo \"$f $(cat $f)\"; done; true")
|
||||
out = docker("exec", container, "sh", "-c", script)
|
||||
values = {}
|
||||
for line in out.splitlines():
|
||||
parts = line.replace(":", " ").split()
|
||||
if parts[0] in ("VmRSS", "VmHWM"):
|
||||
values[parts[0]] = int(parts[1])
|
||||
elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")):
|
||||
values["cgroup_peak"] = int(parts[1])
|
||||
elif parts[0].endswith(("memory.current", "usage_in_bytes")):
|
||||
values["cgroup_current"] = int(parts[1])
|
||||
return values
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
def __init__(self, containers):
|
||||
super().__init__(daemon=True)
|
||||
self.containers = containers
|
||||
self.stop = threading.Event()
|
||||
self.rss = {c: 0 for c in containers}
|
||||
self.cgroup = {c: 0 for c in containers}
|
||||
self.samples = 0
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
for c in self.containers:
|
||||
try:
|
||||
v = proc_status(c)
|
||||
except Exception: # noqa: BLE001 — один пропущенный замер не важен
|
||||
continue
|
||||
self.rss[c] = max(self.rss[c], v.get("VmRSS", 0))
|
||||
self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0))
|
||||
self.samples += 1
|
||||
|
||||
|
||||
def api_containers():
|
||||
from common import api_containers as running
|
||||
return running()
|
||||
|
||||
|
||||
def fresh_replicas():
|
||||
compose("restart", "api")
|
||||
containers = wait_balanced(2, seconds=90)
|
||||
wait_ready()
|
||||
return containers
|
||||
|
||||
|
||||
def warm_up(user, chat):
|
||||
"""По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п."""
|
||||
for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4:
|
||||
upload(user, chat["id"], data, ctype)
|
||||
|
||||
|
||||
def scenario(name, payloads, concurrency):
|
||||
user = new_user("mem")
|
||||
chat = new_chat(user)
|
||||
containers = fresh_replicas()
|
||||
warm_up(user, chat)
|
||||
before = {c: proc_status(c) for c in containers}
|
||||
sampler = Sampler(containers)
|
||||
sampler.start()
|
||||
barrier = threading.Barrier(concurrency)
|
||||
results = [None] * concurrency
|
||||
|
||||
def worker(i):
|
||||
data, ctype = payloads[i % len(payloads)]
|
||||
barrier.wait()
|
||||
started = time.monotonic()
|
||||
try:
|
||||
reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180)
|
||||
results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data)
|
||||
except AssertionError as exc:
|
||||
results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data)
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
|
||||
wall = time.monotonic()
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
wall = time.monotonic() - wall
|
||||
time.sleep(0.5)
|
||||
sampler.stop.set()
|
||||
sampler.join()
|
||||
after = {c: proc_status(c) for c in containers}
|
||||
|
||||
ok = [r for r in results if r[0] == 201]
|
||||
check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 201]}")
|
||||
for status, _, _, att, data in ok[:3]:
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"])
|
||||
check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs")
|
||||
latencies = sorted(r[1] for r in results)
|
||||
per_replica = []
|
||||
for c in containers:
|
||||
b, a = before[c], after[c]
|
||||
per_replica.append({
|
||||
"replica": c[:12],
|
||||
"uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])),
|
||||
"rss_before_mib": round(b["VmRSS"] / 1024, 1),
|
||||
"rss_peak_mib": round(a["VmHWM"] / 1024, 1),
|
||||
"rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1),
|
||||
"rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1),
|
||||
"cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None,
|
||||
"cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None,
|
||||
})
|
||||
sizes = sorted({len(d) for d, _ in payloads})
|
||||
summary = {
|
||||
"scenario": name,
|
||||
"concurrency": concurrency,
|
||||
"file_bytes": sizes,
|
||||
"total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1),
|
||||
"all_201": True,
|
||||
"wall_seconds": round(wall, 2),
|
||||
"latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)},
|
||||
"sampler_rounds": sampler.samples,
|
||||
"replicas": per_replica,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return summary
|
||||
|
||||
|
||||
# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование.
|
||||
NAIVE_VS_STREAMING = r'''
|
||||
import json, os, subprocess, sys
|
||||
code = r"""
|
||||
import io, os, resource, struct, sys, zlib
|
||||
sys.path.insert(0, os.getcwd())
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "x")
|
||||
from PIL import Image
|
||||
import uploads # импорт до замера: сравниваем только саму проверку файла
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d))
|
||||
w, h = 8000, 5000
|
||||
comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3)
|
||||
idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush()
|
||||
data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
|
||||
del idat, comp, row
|
||||
base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
if sys.argv[1] == "naive":
|
||||
body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB)
|
||||
im = Image.open(io.BytesIO(data)); im.load()
|
||||
else:
|
||||
f = io.BytesIO(data); uploads.check_png(f)
|
||||
print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024)
|
||||
"""
|
||||
out = {}
|
||||
for mode in ("naive", "streaming"):
|
||||
r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd())
|
||||
out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:]
|
||||
print(json.dumps(out))
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--output", default="evidence/upload-memory.json")
|
||||
args = parser.parse_args()
|
||||
check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads")
|
||||
|
||||
print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}")
|
||||
print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...")
|
||||
large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)]
|
||||
pixels = [(images.png(8000, 5000), "image/png")]
|
||||
jpeg = [(images.sample_jpeg(), "image/jpeg")]
|
||||
print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes")
|
||||
|
||||
runs = [
|
||||
scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency),
|
||||
scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency),
|
||||
scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency),
|
||||
]
|
||||
naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1])
|
||||
print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) "
|
||||
f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB")
|
||||
limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split()
|
||||
result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20,
|
||||
"api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive}
|
||||
(ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"])
|
||||
print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB "
|
||||
f"(limit {result['api_limit_mib']} MiB); results in {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+519
@@ -0,0 +1,519 @@
|
||||
"""Приём и проверка загружаемого изображения с ограниченной памятью.
|
||||
|
||||
1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше
|
||||
MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном
|
||||
файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса.
|
||||
2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях
|
||||
читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково
|
||||
(CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма),
|
||||
JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import struct
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from typing import BinaryIO
|
||||
|
||||
from PIL import Image
|
||||
from starlette.requests import ClientDisconnect, Request
|
||||
|
||||
import settings
|
||||
from errors import bad_request, payload_too_large, unsupported_media_type
|
||||
|
||||
log = logging.getLogger("messenger.uploads")
|
||||
|
||||
PNG = "image/png"
|
||||
JPEG = "image/jpeg"
|
||||
ALLOWED_TYPES = (PNG, JPEG)
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SIGNATURE = b"\xff\xd8\xff"
|
||||
|
||||
SPOOL_IN_MEMORY = 256 * 1024
|
||||
INFLATE_CHUNK = 64 * 1024
|
||||
CRC_CHUNK = 64 * 1024
|
||||
|
||||
# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке).
|
||||
Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS
|
||||
|
||||
# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY.
|
||||
_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Upload:
|
||||
file: BinaryIO
|
||||
size: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageInfo:
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
class Damaged(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TooManyPixels(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ---------- имя файла и заголовки ----------
|
||||
|
||||
def validate_filename(value: str | None) -> str:
|
||||
"""filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов."""
|
||||
if value is None:
|
||||
raise bad_request("Query parameter filename is required")
|
||||
if not 1 <= len(value) <= 255 or not value.strip():
|
||||
raise bad_request("filename must be 1-255 characters")
|
||||
if value in (".", ".."):
|
||||
raise bad_request("filename is not allowed")
|
||||
for char in value:
|
||||
if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"):
|
||||
raise bad_request("filename contains forbidden characters")
|
||||
return value
|
||||
|
||||
|
||||
def declared_type(header: str | None) -> str:
|
||||
base = (header or "").split(";", 1)[0].strip().lower()
|
||||
if base not in ALLOWED_TYPES:
|
||||
raise unsupported_media_type("Content-Type must be image/png or image/jpeg")
|
||||
return base
|
||||
|
||||
|
||||
def declared_length(header: str | None) -> int | None:
|
||||
if header is None:
|
||||
return None
|
||||
if not header.isdigit():
|
||||
raise bad_request("Invalid Content-Length")
|
||||
length = int(header)
|
||||
if length > settings.MAX_UPLOAD_BYTES:
|
||||
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
|
||||
return length
|
||||
|
||||
|
||||
# ---------- приём тела ----------
|
||||
|
||||
async def receive(request: Request) -> Upload:
|
||||
"""Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется."""
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR)
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS
|
||||
stream = request.stream().__aiter__()
|
||||
try:
|
||||
while True:
|
||||
wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic())
|
||||
if wait <= 0:
|
||||
raise bad_request("Upload took too long")
|
||||
try:
|
||||
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
|
||||
except StopAsyncIteration:
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
raise bad_request("Upload stalled or took too long") from None
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
if size > settings.MAX_UPLOAD_BYTES:
|
||||
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
except ClientDisconnect:
|
||||
spool.close()
|
||||
log.info("Upload aborted by the client after %s bytes, nothing stored", size)
|
||||
raise bad_request("Upload was interrupted") from None
|
||||
except BaseException:
|
||||
spool.close()
|
||||
raise
|
||||
if size == 0:
|
||||
spool.close()
|
||||
raise bad_request("Empty body")
|
||||
return Upload(spool, size, digest.hexdigest())
|
||||
|
||||
|
||||
async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None:
|
||||
"""Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв.
|
||||
|
||||
Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается.
|
||||
"""
|
||||
read = 0
|
||||
deadline = time.monotonic() + seconds
|
||||
try:
|
||||
stream = request.stream().__aiter__()
|
||||
while read <= limit:
|
||||
wait = deadline - time.monotonic()
|
||||
if wait <= 0:
|
||||
return
|
||||
try:
|
||||
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
|
||||
except (StopAsyncIteration, asyncio.TimeoutError):
|
||||
return
|
||||
read += len(chunk)
|
||||
except (ClientDisconnect, RuntimeError):
|
||||
return
|
||||
|
||||
|
||||
# ---------- проверка содержимого ----------
|
||||
|
||||
def sniff(head: bytes) -> str | None:
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return PNG
|
||||
if head.startswith(JPEG_SIGNATURE):
|
||||
return JPEG
|
||||
return None
|
||||
|
||||
|
||||
def inspect(upload: Upload, declared: str) -> ImageInfo:
|
||||
upload.file.seek(0)
|
||||
detected = sniff(upload.file.read(16))
|
||||
if detected is None:
|
||||
raise unsupported_media_type("File is not a PNG or JPEG image")
|
||||
if detected != declared:
|
||||
raise unsupported_media_type("Content-Type does not match the file content")
|
||||
upload.file.seek(0)
|
||||
try:
|
||||
with _decode_slots:
|
||||
if detected == PNG:
|
||||
width, height = check_png(upload.file)
|
||||
upload.file.seek(0)
|
||||
check_png_metadata(upload.file)
|
||||
else:
|
||||
width, height = check_jpeg(upload.file)
|
||||
except TooManyPixels:
|
||||
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
|
||||
except Damaged as exc:
|
||||
log.info("Rejected damaged %s: %s", detected, exc)
|
||||
raise unsupported_media_type("Image is damaged or not supported") from None
|
||||
except Image.DecompressionBombError:
|
||||
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
|
||||
except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc:
|
||||
log.info("Rejected damaged %s: %s", detected, type(exc).__name__)
|
||||
raise unsupported_media_type("Image is damaged or not supported") from None
|
||||
finally:
|
||||
upload.file.seek(0)
|
||||
return ImageInfo(detected, width, height)
|
||||
|
||||
|
||||
def check_pixels(width: int, height: int) -> None:
|
||||
if width < 1 or height < 1:
|
||||
raise Damaged("zero dimension")
|
||||
if width * height > settings.MAX_IMAGE_PIXELS:
|
||||
raise TooManyPixels()
|
||||
|
||||
|
||||
def check_jpeg(file: BinaryIO) -> tuple[int, int]:
|
||||
with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS
|
||||
width, height = image.size
|
||||
check_pixels(width, height)
|
||||
# Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8:
|
||||
# 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB.
|
||||
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
image.load()
|
||||
return width, height
|
||||
|
||||
|
||||
# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR).
|
||||
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
FILTER_BYTES = bytes(range(5))
|
||||
|
||||
|
||||
class PngRows:
|
||||
"""Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки."""
|
||||
|
||||
def __init__(self, width, height, depth, color, interlace):
|
||||
bits = depth * PNG_CHANNELS[color]
|
||||
|
||||
def row_bytes(w):
|
||||
return (w * bits + 7) // 8
|
||||
|
||||
passes = [(width, height)] if interlace == 0 else [
|
||||
((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7]
|
||||
# (длина строки с байтом фильтра, байт в проходе)
|
||||
self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0]
|
||||
self.expected = sum(total for _, total in self.passes)
|
||||
self.index, self.pos = 0, 0
|
||||
self.total = 0
|
||||
|
||||
def feed(self, data: bytes) -> None:
|
||||
view = memoryview(data)
|
||||
self.total += len(data)
|
||||
if self.total > self.expected:
|
||||
raise Damaged("more pixel data than the header says")
|
||||
offset = 0
|
||||
while offset < len(view):
|
||||
stride, size = self.passes[self.index]
|
||||
take = min(len(view) - offset, size - self.pos)
|
||||
first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride
|
||||
if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES):
|
||||
raise Damaged("invalid filter type")
|
||||
self.pos += take
|
||||
offset += take
|
||||
if self.pos == size:
|
||||
self.index, self.pos = self.index + 1, 0
|
||||
|
||||
def done(self) -> bool:
|
||||
return self.total == self.expected
|
||||
|
||||
|
||||
# Чанки, содержимое которых проверяется (держим в памяти; размер ограничен спецификацией):
|
||||
# IHDR — 13 байт, PLTE — до 256 x 3 = 768, tRNS — до 256 (палитра) / 6 (RGB).
|
||||
SMALL_CHUNK_LIMIT = {b"IHDR": 13, b"PLTE": 768, b"tRNS": 256}
|
||||
CRITICAL = (b"IHDR", b"PLTE", b"IDAT", b"IEND")
|
||||
# После данных изображения допустимы только время и текст (строгий список).
|
||||
AFTER_IDAT = {b"IEND", b"tIME", b"tEXt", b"zTXt", b"iTXt"}
|
||||
TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt")
|
||||
# Анимированные PNG (APNG) не поддерживаются: кадры — отдельная сущность, превью строится по одному кадру.
|
||||
APNG_CHUNKS = (b"acTL", b"fcTL", b"fdAT")
|
||||
# Как PngImagePlugin.MAX_TEXT_CHUNK в Pillow: больше 1 MiB распакованного текста в одном чанке — отказ.
|
||||
TEXT_LIMIT = 1024 * 1024
|
||||
# Чанки метаданных до IDAT Pillow (Image.open) читает целиком: ограничиваем, чтобы память не росла.
|
||||
# ICC-профили и EXIF реальных картинок — единицы-сотни KiB.
|
||||
PRE_IDAT_CHUNK_LIMIT = 2 * 1024 * 1024
|
||||
# Чанки, которые по спецификации обязаны стоять ДО первого IDAT (PNG 3rd ed., 5.6 «Chunk ordering»).
|
||||
# После IDAT допустимы только tIME, tEXt, zTXt, iTXt, кадры APNG и частные чанки.
|
||||
BEFORE_IDAT = {b"PLTE", b"tRNS", b"cHRM", b"cICP", b"gAMA", b"iCCP", b"mDCV", b"cLLI", b"sBIT", b"sRGB",
|
||||
b"bKGD", b"hIST", b"pHYs", b"sPLT", b"eXIf"}
|
||||
|
||||
|
||||
def check_png(file: BinaryIO) -> tuple[int, int]:
|
||||
"""Структура PNG по спецификации (PNG 3rd ed., гл. 5 и 11) без декодирования пикселей в память.
|
||||
|
||||
Кроме CRC каждого чанка проверяется СМЫСЛ критичных чанков и tRNS: CRC защищает только от
|
||||
случайной порчи, а намеренно собранный файл с верными CRC может быть некорректным.
|
||||
"""
|
||||
if file.read(8) != PNG_SIGNATURE:
|
||||
raise Damaged("signature")
|
||||
rows = inflater = None
|
||||
width = height = depth = 0
|
||||
color = None
|
||||
palette_entries = 0
|
||||
seen = set() # какие чанки уже встречались
|
||||
idat_started = idat_finished = False
|
||||
first = True
|
||||
while True:
|
||||
header = file.read(8)
|
||||
if len(header) < 8:
|
||||
raise Damaged("truncated: no IEND")
|
||||
length, kind = struct.unpack(">I4s", header)
|
||||
if length > 0x7FFFFFFF:
|
||||
raise Damaged("chunk length")
|
||||
if not kind.isalpha() or not kind.isascii():
|
||||
raise Damaged("chunk type")
|
||||
if first and kind != b"IHDR":
|
||||
raise Damaged("IHDR must be first")
|
||||
if kind[:1].isupper() and kind not in CRITICAL:
|
||||
raise Damaged(f"unknown critical chunk {kind!r}") # такой чанк декодер обязан отвергнуть
|
||||
if kind in SMALL_CHUNK_LIMIT and length > SMALL_CHUNK_LIMIT[kind]:
|
||||
raise Damaged(f"{kind!r} is too long")
|
||||
if kind == b"IDAT":
|
||||
if rows is None or idat_finished:
|
||||
raise Damaged("IDAT chunks must be consecutive")
|
||||
idat_started = True
|
||||
elif idat_started:
|
||||
idat_finished = True
|
||||
if kind in APNG_CHUNKS:
|
||||
raise Damaged("animated PNG is not supported")
|
||||
if kind in BEFORE_IDAT and idat_started:
|
||||
raise Damaged(f"{kind!r} after IDAT")
|
||||
if idat_finished and kind not in AFTER_IDAT:
|
||||
raise Damaged(f"{kind!r} after IDAT")
|
||||
if not idat_started and kind != b"IDAT" and length > PRE_IDAT_CHUNK_LIMIT:
|
||||
raise Damaged(f"{kind!r} metadata chunk is too large")
|
||||
if kind == b"tIME" and length != 7:
|
||||
raise Damaged("tIME size")
|
||||
if kind in (b"IHDR", b"PLTE", b"tRNS", b"IEND") and kind in seen:
|
||||
raise Damaged(f"duplicate {kind!r}")
|
||||
seen.add(kind)
|
||||
|
||||
crc = zlib.crc32(kind)
|
||||
kept = b""
|
||||
text = TextChunk(kind) if kind in TEXT_CHUNKS else None
|
||||
left = length
|
||||
while left:
|
||||
piece = file.read(min(CRC_CHUNK, left))
|
||||
if not piece:
|
||||
raise Damaged("truncated chunk")
|
||||
left -= len(piece)
|
||||
crc = zlib.crc32(piece, crc)
|
||||
if kind in SMALL_CHUNK_LIMIT:
|
||||
kept += piece
|
||||
elif kind == b"IDAT":
|
||||
feed_inflate(inflater, rows, piece)
|
||||
elif text is not None:
|
||||
text.feed(piece)
|
||||
if text is not None:
|
||||
text.finish()
|
||||
stored = file.read(4)
|
||||
if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF:
|
||||
raise Damaged(f"CRC mismatch in {kind!r}")
|
||||
|
||||
if kind == b"IHDR":
|
||||
if length != 13:
|
||||
raise Damaged("IHDR")
|
||||
width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept)
|
||||
if width > 0x7FFFFFFF or height > 0x7FFFFFFF:
|
||||
raise Damaged("dimensions")
|
||||
check_pixels(width, height) # до распаковки
|
||||
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]:
|
||||
raise Damaged("color type / bit depth")
|
||||
if compression != 0 or filter_method != 0 or interlace not in (0, 1):
|
||||
raise Damaged("IHDR methods")
|
||||
rows = PngRows(width, height, depth, color, interlace)
|
||||
inflater = zlib.decompressobj()
|
||||
elif kind == b"PLTE":
|
||||
# 1-256 записей по 3 байта; для палитровых — не больше 2^глубина; у серых палитры быть не может.
|
||||
if color in (0, 4):
|
||||
raise Damaged("PLTE in a grayscale image")
|
||||
if length == 0 or length % 3:
|
||||
raise Damaged("palette size")
|
||||
palette_entries = length // 3
|
||||
if color == 3 and palette_entries > 1 << depth:
|
||||
raise Damaged("palette larger than the bit depth allows")
|
||||
elif kind == b"tRNS":
|
||||
# Прозрачность: серое — 2 байта, RGB — 6, палитра — не больше записей PLTE; при альфа-канале запрещён.
|
||||
if color in (4, 6):
|
||||
raise Damaged("tRNS with an alpha channel")
|
||||
if color == 3 and not palette_entries:
|
||||
raise Damaged("tRNS before PLTE")
|
||||
expected_ok = {0: length == 2, 2: length == 6, 3: 1 <= length <= palette_entries}[color]
|
||||
if not expected_ok:
|
||||
raise Damaged("tRNS size")
|
||||
elif kind == b"IEND":
|
||||
if length != 0:
|
||||
raise Damaged("IEND with data")
|
||||
break
|
||||
first = False
|
||||
|
||||
if file.read(1):
|
||||
raise Damaged("data after IEND")
|
||||
if color == 3 and not palette_entries:
|
||||
raise Damaged("palette image without PLTE")
|
||||
if not idat_started:
|
||||
raise Damaged("no IDAT")
|
||||
tail = inflater.flush(INFLATE_CHUNK)
|
||||
if tail:
|
||||
rows.feed(tail)
|
||||
if not inflater.eof or inflater.unused_data or not rows.done():
|
||||
raise Damaged("pixel data is incomplete")
|
||||
return width, height
|
||||
|
||||
|
||||
def check_png_metadata(file: BinaryIO) -> None:
|
||||
"""Второе мнение: стандартный парсер Pillow разбирает метаданные до IDAT (gAMA, iCCP, pHYs, tEXt…),
|
||||
НЕ читая пиксели. Только open, без verify(): verify читает каждый чанк, включая IDAT до 10 MiB,
|
||||
целиком в память, а CRC и данные после IDAT уже потоково проверил check_png.
|
||||
Нужен для вспомогательных чанков: по спецификации их можно игнорировать, но если Pillow
|
||||
не может их разобрать, файл нельзя будет обработать (превью в lab5) — лучше отклонить сразу.
|
||||
"""
|
||||
with Image.open(file, formats=["PNG"]):
|
||||
pass # open разбирает все чанки до первого IDAT; после IDAT check_png пускает только время и текст
|
||||
|
||||
|
||||
class TextChunk:
|
||||
"""Потоковая проверка tEXt / zTXt / iTXt: ключ 1-79 байт, метод сжатия 0, сжатый текст —
|
||||
корректный zlib не больше TEXT_LIMIT после распаковки (распаковка кусками, текст не хранится)."""
|
||||
|
||||
def __init__(self, kind: bytes):
|
||||
self.kind = kind
|
||||
self.head = b""
|
||||
self.done_header = False
|
||||
self.inflater = None
|
||||
self.size = 0
|
||||
|
||||
def feed(self, piece: bytes) -> None:
|
||||
if not self.done_header:
|
||||
self.head += piece
|
||||
rest = self._header()
|
||||
if rest is None:
|
||||
if len(self.head) > 4096:
|
||||
raise Damaged("text chunk header")
|
||||
return
|
||||
piece = rest
|
||||
if self.inflater is not None and piece:
|
||||
self._inflate(piece)
|
||||
|
||||
def _keyword_end(self) -> int | None:
|
||||
end = self.head.find(b"\x00")
|
||||
if end == -1:
|
||||
return None
|
||||
if not 1 <= end <= 79:
|
||||
raise Damaged("text keyword")
|
||||
return end
|
||||
|
||||
def _header(self) -> bytes | None:
|
||||
end = self._keyword_end()
|
||||
if end is None:
|
||||
return None
|
||||
h = self.head
|
||||
if self.kind == b"tEXt":
|
||||
self.done_header = True
|
||||
return b""
|
||||
if self.kind == b"zTXt":
|
||||
if len(h) < end + 2:
|
||||
return None
|
||||
if h[end + 1] != 0:
|
||||
raise Damaged("zTXt compression method")
|
||||
self.done_header, self.inflater = True, zlib.decompressobj()
|
||||
return h[end + 2:]
|
||||
# iTXt: keyword \0 flag method language \0 translated-keyword \0 text
|
||||
if len(h) < end + 3:
|
||||
return None
|
||||
flag, method = h[end + 1], h[end + 2]
|
||||
if flag not in (0, 1) or (flag and method != 0):
|
||||
raise Damaged("iTXt compression")
|
||||
lang_end = h.find(b"\x00", end + 3)
|
||||
tkey_end = h.find(b"\x00", lang_end + 1) if lang_end != -1 else -1
|
||||
if tkey_end == -1:
|
||||
return None
|
||||
self.done_header = True
|
||||
if flag:
|
||||
self.inflater = zlib.decompressobj()
|
||||
return h[tkey_end + 1:]
|
||||
|
||||
def _inflate(self, data: bytes) -> None:
|
||||
if self.inflater.eof:
|
||||
raise Damaged("data after compressed text")
|
||||
out = self.inflater.decompress(data, INFLATE_CHUNK)
|
||||
self.size += len(out)
|
||||
while self.inflater.unconsumed_tail and not self.inflater.eof and self.size <= TEXT_LIMIT:
|
||||
out = self.inflater.decompress(self.inflater.unconsumed_tail, INFLATE_CHUNK)
|
||||
self.size += len(out)
|
||||
if self.size > TEXT_LIMIT:
|
||||
raise Damaged("compressed text is too large")
|
||||
if self.inflater.eof and self.inflater.unused_data:
|
||||
raise Damaged("data after compressed text")
|
||||
|
||||
def finish(self) -> None:
|
||||
if not self.done_header:
|
||||
raise Damaged("text chunk header")
|
||||
if self.inflater is not None:
|
||||
self.size += len(self.inflater.flush(INFLATE_CHUNK))
|
||||
if not self.inflater.eof or self.size > TEXT_LIMIT:
|
||||
raise Damaged("compressed text")
|
||||
|
||||
|
||||
def feed_inflate(inflater, rows: PngRows, data: bytes) -> None:
|
||||
"""Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память."""
|
||||
if inflater.eof:
|
||||
if data:
|
||||
raise Damaged("data after the end of zlib stream")
|
||||
return
|
||||
out = inflater.decompress(data, INFLATE_CHUNK)
|
||||
rows.feed(out)
|
||||
while inflater.unconsumed_tail and not inflater.eof:
|
||||
out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK)
|
||||
rows.feed(out)
|
||||
if inflater.eof and inflater.unused_data:
|
||||
raise Damaged("data after the end of zlib stream")
|
||||
Reference in New Issue
Block a user