ЛР 4: S3-хранилище (SeaweedFS) и вложения

This commit is contained in:
kalibr456
2026-10-21 21:25:10 +03:00
parent ec540a1a72
commit 5c3a31db2e
81 changed files with 6847 additions and 25 deletions
+7
View File
@@ -18,3 +18,10 @@ __pycache__
node_modules
evidence
.DS_Store
tests
student-tests
contracts
scripts
infra
examples
*.md
+25 -2
View File
@@ -7,14 +7,37 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Pools and timeouts (same as lab3)
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_POOL_WAIT_SECONDS=3
DB_CONNECT_TIMEOUT_SECONDS=3
DB_STATEMENT_TIMEOUT_MS=5000
REDIS_POOL_MAX=20
REDIS_TIMEOUT_MS=500
SHUTDOWN_DRAIN_SECONDS=3
SHUTDOWN_TIMEOUT_SECONDS=10
# lab4: S3 client and uploads
S3_CONNECT_TIMEOUT_SECONDS=1
S3_READ_TIMEOUT_SECONDS=5
S3_MAX_ATTEMPTS=2
S3_POOL_MAX=20
UPLOAD_TIMEOUT_SECONDS=60
UPLOAD_IDLE_TIMEOUT_SECONDS=10
DECODE_CONCURRENCY=2
# Cleanup of abandoned uploads (janitor service): grace period and pass interval
ATTACHMENT_GRACE_SECONDS=3600
JANITOR_INTERVAL_SECONDS=300
+6
View File
@@ -0,0 +1,6 @@
*.sh text eol=lf
*.sql text eol=lf
*.conf text eol=lf
*.cfg text eol=lf
*.example text eol=lf
*.cmd text eol=crlf
+8 -8
View File
@@ -1,14 +1,14 @@
## Лабораторная 4
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Бабиченко Максим, БВТ2303
Целевая оценка (3/4/5): 4
Трек 5, если заявлен: не заявлен
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено: перенесён весь lab3 (API на FastAPI, PostgreSQL, сессии в Redis, HAProxy + 2 реплики, идемпотентность, migrate, свои тесты). Добавлены: SeaweedFS (`s3`) с приватным bucket и volume `s3data` (MinIO недоступен в публичных реестрах); идемпотентный bootstrap `s3-init`; отдельный ключ API с правами только на свой bucket; миграция 004 (`attachments`, `message_attachments` с составными FK по `chat_id`); синхронная загрузка PNG/JPEG с потоковыми лимитами (10 MiB, 40 Мпикс до распаковки, MIME по содержимому, проверка целостности без декодирования в память), скачивание через API с проверкой участника; `attachment_ids` в сообщениях; запись «строка-намерение -> PUT -> ready» и сервис `janitor` (очистка брошенных загрузок и объектов-сирот с grace period, dry-run, отчёт о missing). Подробности — REPORT.md.
Команды запуска из чистого clone: `Copy-Item .env.example .env; python scripts/init_secrets.py; docker compose up -d --build --wait; python student-tests/collect_evidence.py` (полностью — REPORT.md, раздел «Запуск из чистого clone»).
Результаты проверок и ссылка на REPORT.md: evidence/summary.md и evidence/*.txt; таблицы замеров — REPORT.md, раздел «Эксперименты и отказы».
Ограничения / известные ошибки: один узел (lb, postgres, redis, s3 — SPOF); повтор загрузки после потерянного ответа создаёт второе вложение (так по контракту); missing-объекты janitor только находит. См. REPORT.md, «Ограничения».
Источники и использование ИИ: библиотеки и документация перечислены в REPORT.md; при написании кода и тестов использовался ИИ-ассистент (Claude), код проверен и прогнан мной.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
+1
View File
@@ -15,6 +15,7 @@ data/
backups/
# Runtime/build artifacts
.venv/
venv/
__pycache__/
*.py[cod]
node_modules/
+27
View File
@@ -0,0 +1,27 @@
FROM python:3.11.13-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt
COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py ./
COPY migrations/ ./migrations/
RUN useradd --create-home --shell /usr/sbin/nologin appuser
USER appuser
ARG APP_VERSION=dev
ENV APP_VERSION=${APP_VERSION}
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)"
CMD ["python", "main.py"]
+213 -15
View File
@@ -1,37 +1,235 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Бабиченко Максим, БВТ2303
- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран
- Commit/PR: <<ссылка на MR в Gitea после публикации>>
- Использованные источники, библиотеки и помощь ИИ: всё из lab3 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py; образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine, haproxy:3.2.25-alpine). Новое: boto3/botocore 1.43.116 (S3-клиент), Pillow 12.3.0 (проверка JPEG), образ chrislusf/seaweedfs:3.80. Документация: AWS S3 (PutObject, x-amz-checksum-sha256, presigned URLs), SeaweedFS (`weed server -s3`, S3 API, статический конфиг identities с правами `Read/Write/List:<bucket>`), спецификация PNG (структура чанков, CRC, IHDR, фильтры строк, Adam7), botocore Config (timeouts, `total_max_attempts`). При написании кода и тестов использовался ИИ-ассистент (Claude); весь код проверен и прогнан мной, объяснить и изменить его могу на защите.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Проверял на Windows 11, все команды в PowerShell. Версии Docker, Compose и ресурсы машины — в `evidence/stack.txt`. Python нужен только для тестов (стандартная библиотека, Pillow на хосте не нужен). Перед запуском остановить стенд lab3 (`docker compose down` в его папке), иначе занят порт 8080.
```powershell
git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab4.git
cd mtusi2026lab4
git checkout solution/bvt2303-babichenko
$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают
Copy-Item .env.example .env
python scripts/init_secrets.py # случайные пароли, S3-ключи и secrets/s3_config.json (папка в .gitignore)
docker compose up -d --build --wait # postgres, s3 -> migrate, s3-init -> 2 реплики api, janitor -> lb
docker compose ps -a # migrate и s3-init: Exited (0), остальные healthy/running
python student-tests/collect_evidence.py # все проверки по очереди (~8 мин), вывод в evidence/, таблицы в этот отчёт
```
Отдельные проверки (make на моём Windows нет, ниже то же самое без make):
| Команда из README | То же в PowerShell |
| --- | --- |
| `make check` | `python scripts/check_contract.py` |
| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` |
| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` |
| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` |
| тесты вложений (уровень 3–4) | `python student-tests/test_attachments.py` |
| S3 вне API, права ключа, bootstrap | `python student-tests/test_s3_security.py` |
| отказы S3/БД, janitor | `python student-tests/test_consistency.py [s3\|crash\|orphan\|aborted]` |
| 10+ одновременных загрузок, память | `python student-tests/upload_memory.py` |
| ручной проход очистки | `docker compose exec janitor python janitor.py --once --dry-run` |
После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть её: `docker start <ID из docker compose ps -a>` или `docker compose up -d --wait`. На Linux/macOS вместо `.cmd` — `scripts/restart.sh` и `scripts/stop-one.sh`.
Bootstrap: миграции применяет одноразовый сервис `migrate`, bucket и проверку прав S3 — одноразовый сервис `s3-init` (`s3_init.py`, тот же образ, что у API); оба запускаются сами при `docker compose up`, оба идемпотентны. Повторить вручную: `docker compose run --rm migrate`, `docker compose run --rm s3-init`. Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py` (новые пароли не подходят к старым volumes, поэтому удаляются вместе).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
Клиент --HTTP--> lb: HAProxy 3.2, 127.0.0.1:8080 (единственный опубликованный порт)
| round robin, без привязки клиента к реплике
+--> api-1 --+
+--> api-2 --+--> postgres: пользователи, чаты, сообщения, МЕТАДАННЫЕ вложений (volume pgdata)
+--> redis: сессии (AOF, volume redisdata)
+--> s3 (SeaweedFS): БАЙТЫ вложений, приватный bucket (volume s3data)
janitor: раз в 5 мин удаляет брошенные загрузки и объекты-сироты (postgres + s3)
migrate: одноразово применяет migrations/*.sql
s3-init: одноразово создаёт приватный bucket и проверяет права ключей и закрытость bucket
сети: edge (lb <-> api) и data (всё остальное; internal, без выхода наружу)
```
| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` на 8404 | нет | edge; `127.0.0.1:8080`, `127.0.0.1:8404/stats` | не нужны |
| API ×2 | `api` (Dockerfile, `deploy.replicas: 2`) | python:3.11.13-slim | `/health/ready` (PostgreSQL + Redis) | нет | edge + data; наружу не публикуется | `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` |
| S3 | `s3` | chrislusf/seaweedfs:3.80 (`weed server -s3`: master, volume, filer и S3 в одном процессе) | S3-порт отвечает по HTTP (`wget`) | `s3data` (данные и метаданные filer) | только data; S3 8333, UI master 9333 и filer 8888 доступны только внутри стенда, наружу не публикуются | `s3_config` (конфиг ключей и прав) |
| Bootstrap S3 | `s3-init` (образ приложения, `s3_init.py`) | python:3.11.13-slim | завершается с кодом 0 | нет | data | admin-ключ + ключ API (для проверки прав) |
| Очистка | `janitor` (тот же образ) | python:3.11.13-slim | процесс работает | нет | data | `database_url` + ограниченный ключ S3 |
| Миграции | `migrate` | python:3.11.13-slim | завершается с кодом 0 | нет | data | только `database_url` |
| База данных | `postgres` | postgres:17.11-alpine | `pg_isready` | `pgdata` | data, 5432 | `POSTGRES_PASSWORD_FILE` |
| Сессии | `redis` | redis:7.4.11-alpine | `redis-cli ping` с паролем | `redisdata` (AOF) | data, 6379 | `include /run/secrets/redis_auth` |
Почему SeaweedFS: из трёх разрешённых вариантов MinIO отпал — его образы больше не раздаются ни с Docker Hub, ни с quay.io (проверено при сборке стенда: `pull access denied` / `unauthorized`), а закреплённый тег без доступного образа не воспроизводится. У SeaweedFS образ на Docker Hub, а права ключей задаются статическим конфигом (`-s3.config`): отдельный ключ API с правами только на свой bucket получается без отдельной утилиты администрирования. Закреплена версия 3.80, `latest` не используется.
Что перенесено из lab3 без изменений по смыслу: аутентификация и сессии в Redis, пользователи/чаты/сообщения, идемпотентность, HAProxy с проверкой готовности и аварийным режимом, двухшаговая остановка по SIGTERM, сервис `migrate`, свои тесты lab1–3.
Что изменено в lab4:
- Новые файлы: `storage.py` (S3-клиент), `uploads.py` (приём и проверка файла), `attachments.py` (запись в две системы), `janitor.py` (очистка/сверка), `s3_init.py` (bootstrap S3), `migrations/004_attachments.sql`.
- `main.py`: endpoint загрузки, метаданных и скачивания; `attachment_ids` в сообщениях; `db.py`: вложения и связь сообщение–вложение.
- `compose.yaml`: сервисы `s3`, `s3-init`, `janitor`, volume `s3data`, пять новых секретов (admin-ключ, ключ API, конфиг ключей SeaweedFS); `infra/haproxy.cfg`: `timeout server 30s` (см. ниже); restart-скрипты перезапускают ещё `s3` и `janitor`.
- `scripts/init_secrets.py` дописывает недостающие секреты (в том числе `s3_config.json` с правами ключей) и не трогает существующие.
### Модель данных
`attachments` (миграция 004): `id`, `chat_id`, `owner_id`, `filename` (только для отображения), `content_type`, `size_bytes`, `sha256`, `width`, `height`, `object_key`, `state` (`uploading` | `ready`), `created_at`, `updated_at`. `object_key` = `originals/<32 случайных hex>`: его назначает сервер, имя файла в ключ не попадает (CHECK на формат). Внешний ключ `(chat_id, owner_id) -> chat_members`: загрузить можно только в свой чат.
`message_attachments (message_id, chat_id, attachment_id, position)`: порядок вложений в сообщении. Оба внешних ключа составные и содержат один и тот же `chat_id`: `(message_id, chat_id) -> messages`, `(attachment_id, chat_id) -> attachments`. Поэтому вложение другого чата нельзя прикрепить даже прямым INSERT в обход API. Одно вложение можно прикрепить к нескольким сообщениям (контракт: «любой участник может ссылаться на доступное вложение чата»). Ограничение на текст сообщения ослаблено до `<= 16000 байт`: пустой текст допустим с вложением, проверку «текст или вложение» делает API.
### Загрузка: что и в каком порядке проверяется
`POST /api/v1/chats/{chat_id}/attachments?filename=...`, тело — сырые байты одного PNG/JPEG.
1. До чтения тела: сессия (401) -> участник чата (404) -> `filename` 1–255 символов без `/`, `\`, NUL, управляющих символов, не `.`/`..` (400) -> `Content-Type` только `image/png`/`image/jpeg` (415) -> `Content-Length` больше 10 485 760 (413). При этих ошибках тело дочитывается и выбрасывается (до 20 MiB / 10 с, ничего не сохраняя), чтобы клиент получил ответ, а не обрыв соединения.
2. Тело читается кусками: размер и SHA-256 считаются на лету, данные пишутся во временный файл (до 256 KiB в памяти, остальное на диск контейнера). Первый байт сверх 10 MiB -> 413, чтение прекращается (работает и без `Content-Length`, при `Transfer-Encoding: chunked`). Таймауты: 10 с без данных, 60 с на всё тело (`UPLOAD_IDLE_TIMEOUT_SECONDS`, `UPLOAD_TIMEOUT_SECONDS`).
3. Тип по сигнатуре (`\x89PNG...`, `FF D8 FF`), не по заголовку. Не PNG/JPEG -> 415; заявленный тип не совпадает с содержимым -> 415.
4. Размер в пикселях читается из заголовка (IHDR / SOF) и проверяется ДО распаковки: больше 40 000 000 -> 413.
5. Проверка целостности с ограниченной памятью:
- PNG — своя потоковая проверка (`uploads.check_png`): CRC каждого чанка, порядок IHDR/PLTE/IDAT/IEND, допустимые тип цвета и глубина, ничего после IEND; IDAT распаковывается zlib кусками по 64 KiB (вывод не хранится), считается точный объём данных по IHDR (с учётом Adam7) и проверяется байт фильтра каждой строки. «Zip-бомба» не раздувает память: распаковка идёт порциями и обрывается, как только данных больше, чем обещал заголовок.
- JPEG — Pillow декодирует весь поток, но в масштабе 1/8 (`draft`): 40 Мпикс -> ~0,6 Мпикс, около 2 MiB.
- Одновременно в реплике проверяется не больше `DECODE_CONCURRENCY=2` файлов (CPU).
Битый файл -> 415. Любая ошибка разбора ловится и превращается в 415, а не 500. Проверка PNG сверена с Pillow: 400 случайных корректных PNG разных типов приняты, 400 испорченных (обрезка, перевёрнутый бит) отклонены.
6. Запись (см. следующий раздел) и ответ `201 Attachment` со `status=ready`, `variants=[]`, `error_code=null`.
Временный файл удаляется в `finally` при любом исходе: оригинал после ответа существует только в S3. У контейнеров `api` нет volumes, это проверяет `test_consistency.py`.
Скачивание: `GET /api/v1/attachments/{id}/content` — сначала проверка участника чата в PostgreSQL (иначе 404, как у несуществующего), потом GET из S3 и потоковая отдача кусками по 64 KiB с истинным `Content-Type`, `Content-Length`, `X-Content-Type-Options: nosniff`, `Content-Disposition: inline; filename*=UTF-8''...`. Без redirect и без подписанных URL. `variant` — только `original` (иное -> 400).
### Согласованность PostgreSQL и S3
Распределённой транзакции нет, поэтому порядок записи выбран так, чтобы у каждого объекта в bucket была строка-указатель в БД (`attachments.py`):
| Шаг | Если здесь отказ |
| --- | --- |
| 1. `INSERT attachments (state='uploading')` + COMMIT | в S3 ничего нет; клиенту 503 |
| 2. `PUT` объекта в S3 с `x-amz-checksum-sha256` (S3 сам сверяет SHA-256) | откат: короткий DELETE объекта (PUT мог пройти при таймауте) и удаление строки; если S3 недоступен и откат не удался, строка `uploading` остаётся указателем для janitor; клиенту 503 |
| 3. `UPDATE state='ready'` + COMMIT | клиенту 503; объект сразу НЕ удаляется: при обрыве связи COMMIT мог пройти, удаление сломало бы готовое вложение. Строка остаётся `uploading` -> её разберёт janitor |
| 4. `201` | только после шага 3 |
Строки `uploading` не видны через API (GET -> 404) и не прикрепляются к сообщениям (404), сообщение может ссылаться только на `ready` (проверка под `FOR SHARE` в той же транзакции, что и INSERT сообщения).
`janitor.py` (сервис `janitor`, раз в `JANITOR_INTERVAL_SECONDS=300`, один экземпляр за счёт `pg_try_advisory_lock`):
1. Строки `uploading` старше grace period: под `FOR UPDATE SKIP LOCKED` удалить объект (DELETE идемпотентен), затем строку. Если S3 недоступен — транзакция откатывается, строка остаётся до следующего прохода.
2. Обход bucket (`originals/`, постранично): объект старше grace period без строки в БД (в любом состоянии) — сирота, удаляется (перед удалением ещё раз проверяется, что строка не появилась). Живая загрузка всегда имеет строку, потому что строка коммитится до PUT.
3. Отчёт о `ready`-строках без объекта (`missing_objects`). Их janitor не удаляет: это потеря данных, её чинят восстановлением из резервной копии, а не автоматической очисткой; скачивание такого файла отвечает 503, не 500.
Grace period: `ATTACHMENT_GRACE_SECONDS=3600`, janitor и API отказываются стартовать, если он меньше `2 x UPLOAD_TIMEOUT_SECONDS`. Самая долгая загрузка после записи строки живёт: PUT до 2 x (1 + 5) с + откат 2 с + UPDATE до 8 с, то есть меньше 30 с — в 120 раз меньше grace period. Поэтому janitor не может удалить загрузку, которая ещё идёт, а если бы реплика зависла дольше часа, её UPDATE не найдёт строку (`state='uploading'` уже удалена под блокировкой) и клиент получит 503, а не «ready без объекта». Есть `--dry-run` (только отчёт) и `--once`.
Обрыв загрузки клиентом: строка пишется только после приёма и проверки всего тела, поэтому оборванная загрузка не оставляет ни строки, ни объекта; временный файл удаляется сразу.
### Готовность и отказ S3
S3 не входит в `/health/ready`: без него не работают только загрузка и скачивание файлов, а вход, чаты, сообщения (в том числе со ссылкой на уже загруженные вложения) и метаданные вложений продолжают работать. Если бы readiness зависел от S3, отказ S3 вывел бы из балансировки все реплики и остановил бы весь мессенджер. При недоступном S3 файловые операции отвечают `503 unavailable` без подробностей. Критичны для readiness, как и в lab3, PostgreSQL и Redis.
S3-клиент: connect timeout 1 с, read timeout 5 с, всего 2 попытки (`total_max_attempts`), откат — 1 попытка с таймаутом 2 с. Худший ответ при зависшем S3 около 13 с, поэтому в HAProxy `timeout server` увеличен с 15 до 30 с (пока клиент передаёт тело файла, этот таймаут не срабатывает: передачу ограничивают таймауты самого API).
### Безопасность хранилища
- Bucket приватный: в конфиге ключей SeaweedFS нет identity `anonymous`, поэтому любой запрос без подписи получает 403. Проверка вне API: `s3-init` и `test_s3_security.py` делают анонимные GET/HEAD/LIST/PUT прямо в S3 из внутренней сети — 403.
- Два ключа (`secrets/s3_config.json`, генерирует `scripts/init_secrets.py`): `admin` (Admin/Read/Write/List) есть только у `s3` (в конфиге) и `s3-init`; `messenger-api` для api и janitor — только `Read:messenger-attachments`, `Write:messenger-attachments`, `List:messenger-attachments`. Создать/удалить bucket, поменять политику или versioning ключ API не может — проверяют `s3-init` и `test_s3_security.py`.
- Порты S3 (8333), master (9333) и filer (8888) не публикуются, SeaweedFS только во внутренней сети `data` (`internal: true`); веб-интерфейсы master/filer доступны только контейнерам стенда, с хоста — connection refused.
- Bootstrap идемпотентен: `create_bucket` при существующем bucket получает `BucketAlreadyOwnedByYou/BucketAlreadyExists` и продолжает; проверочный объект ключа API создаётся и удаляется; в конце скрипт проверяет, что bucket private. Повторный `docker compose run --rm s3-init` ничего не ломает (тест). Права ключей — декларативный конфиг, применяется при старте `s3`.
- Приватный bucket и случайный ключ не заменяют проверку доступа в API: ключ объекта не секрет (может попасть в лог, бэкап, ошибку), а bucket «приватен» для всех, у кого нет ключа S3, — но не различает пользователей мессенджера. Кто участник какого чата, знает только API, поэтому проверка участника делается на каждый GET метаданных и байтов.
### Соответствие `.env.example`
| Переменная | Что делает в этом решении |
| --- | --- |
| APP_HOST, APP_PORT, APP_ENV | где слушает API (0.0.0.0:8080 внутри контейнера), APP_ENV пишется в лог |
| SESSION_TTL_SECONDS, SESSION_STORE=redis | срок жизни сессии 3600 с, сессии в Redis (как в lab3) |
| DATABASE_URL_FILE, CACHE_URL_FILE | пути к секретам с адресами PostgreSQL и Redis |
| API_REPLICAS | число реплик `api`, по умолчанию 2 |
| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) |
| S3_ENDPOINT, S3_REGION, S3_BUCKET | `http://s3:8333`, `us-east-1`, `messenger-attachments` (path-style адресация) |
| S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE | ограниченный ключ API (identity `messenger-api` в `s3_config.json`) |
| MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS | 10 485 760 байт и 40 000 000 пикселей |
| S3_CONNECT_TIMEOUT_SECONDS, S3_READ_TIMEOUT_SECONDS, S3_MAX_ATTEMPTS, S3_POOL_MAX | 1 с, 5 с, 2 попытки всего, 20 соединений |
| UPLOAD_TIMEOUT_SECONDS, UPLOAD_IDLE_TIMEOUT_SECONDS | 60 с на всё тело, 10 с без данных |
| DECODE_CONCURRENCY | одновременных проверок изображений в реплике, 2 |
| ATTACHMENT_GRACE_SECONDS, JANITOR_INTERVAL_SECONDS | grace period 3600 с, проход janitor раз в 300 с |
| DB_*, REDIS_*, SHUTDOWN_* | пулы, таймауты и остановка (как в lab3) |
## Доказательства критериев
Все файлы `evidence/*.txt` создаёт `python student-tests/collect_evidence.py` (вывод тестов как есть, с датой, commit и кодом выхода); сводка — `evidence/summary.md`.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS через балансировщик, с загрузкой, сообщением с `attachment_ids`, скачиванием и 404 постороннему | evidence/smoke.txt |
| Skeleton и контракт не изменены | `python scripts/check_contract.py` | PASS; `contracts/`, `tests/`, README не менялись | evidence/check.txt |
| 3: возможности lab3 сохранены | `test_balancing.py`, `test_idempotency.py`, `test_api.py`, `test_race.py`, `test_db.py`, `redis_info.py`, `explain.py`, `test_failures.py`, публичный failover | все PASS через балансировщик | evidence/balancing.txt, idempotency.txt, student-tests.txt, race.txt, db.txt, redis.txt, explain.txt, failures.txt, failover.txt |
| 3: S3 в общем стенде; метаданные в PostgreSQL, байты в приватном bucket, не в БД и не в каталоге API | `check_stack.py`, `test_s3_security.py`, `test_consistency.py aborted` | `s3` и `s3-init` в compose; у `api` нет volumes, временных файлов после запросов 0; анонимный доступ к bucket 403 | evidence/stack.txt, s3-security.txt, consistency.txt |
| 3: POST -> 201 ready после сохранения; оригинал побайтно; message с attachment_ids читается участниками; посторонний без доступа | smoke, `test_attachments.py` (roundtrip, access, message) | SHA-256 и размер совпадают, скачивает второй участник, посторонний 404 на метаданные и байты | evidence/smoke.txt, attachments.txt |
| 3: реальные PNG/JPEG ≤10 MiB и ≤40 Мпикс; MIME по содержимому; опасные filename; лимиты до неограниченного чтения/декодирования | `test_attachments.py` | ровно 10 MiB -> 201, +1 байт -> 413 (с Content-Length и chunked); 8000x5000 -> 201, 8001x5000 -> 413, «бомба» 100000x100000 -> 413 по заголовку; 13 опасных имён -> 400 | evidence/attachments.txt |
| 3: upload -> остановка обслужившей реплики -> download через другую | публичный `make test-failover` (lab4 загружает файл до отказа и скачивает после) для каждой реплики; `test_balancing.py` | PASS для обеих реплик; файл, загруженный через одну реплику, скачан через другую | evidence/failover.txt, balancing.txt |
| 3: restart с сохранением volumes сохраняет метаданные и оригинал | `run_persistence.py` (публичный persistence + ID/время старта контейнеров) | postgres, redis, s3, обе api, janitor, lb перезапущены (те же контейнеры, новое время старта); файл скачивается с прежним токеном | evidence/persistence.txt |
| 4: тесты: слишком большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; нет 500 | `test_attachments.py` | 413; 10 видов битых файлов -> 415; 7 подмен типа -> 415; неизвестный id и вложение другого чата (автор состоит в обоих) -> 404; дубли, 11 id -> 400; ни одного 500 | evidence/attachments.txt |
| 4: прерывание загрузки и отказ между S3/БД не оставляют вечных неучтённых объектов; механизм с grace period | `test_consistency.py` (aborted, crash, orphan, s3) | обрыв: строк и объектов не прибавилось; падение после PUT: строка `uploading` невидима, janitor не трогает её до grace, `--dry-run` только отчёт, затем удалены объект и строка, повтор — 0; сирота удаляется только после grace; `ready` не трогаются; missing — только отчёт | evidence/consistency.txt |
| 4: недоступность S3 | `test_consistency.py s3` (`docker compose stop s3` и `pause s3`) | файловые операции 503 без подробностей, остальное работает; после восстановления файлы на месте, мусор убран | evidence/consistency.txt |
| 4: лимиты/таймауты и потоковая обработка; ≥10 одновременных загрузок с пиком памяти | `upload_memory.py` | таблица ниже | evidence/upload-memory.txt, upload-memory.json |
| 4: bootstrap идемпотентен | `test_s3_security.py` (повторный `docker compose run --rm s3-init`) | exit 0, «bucket is private», старые файлы скачиваются, новая загрузка 201 | evidence/s3-security.txt |
| 4: отдельные S3 credentials с ограниченными правами | `test_s3_security.py`, `check_stack.py`, `s3-init` | ключ API: get/put/delete/list своего bucket — да; create/delete bucket, policy, versioning — отказ; admin-ключ и конфиг ключей только у s3 и s3-init | evidence/s3-security.txt, stack.txt |
| 4: консоль не открыта; проверка private bucket вне API | `test_s3_security.py`, логи `s3-init` | 127.0.0.1:8333/9333/8888 — connection refused, s3 только в сети data; анонимные GET/HEAD/LIST/PUT/policy прямо в S3 -> 403 | evidence/s3-security.txt |
| Выбранный трек 5 | — | не выбран | — |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Гипотезы:
1. Память API-реплики почти не зависит от числа и размера одновременных загрузок: тело идёт во временный файл, PNG проверяется потоково, JPEG — в масштабе 1/8, проверок одновременно не больше 2 на реплику. Наивный вариант (тело в памяти + полное декодирование 40 Мпикс ~160 MiB) при 10 одновременных загрузках не поместился бы в лимит 512 MiB.
2. Отказ S3 затрагивает только операции с байтами файлов и не оставляет видимых или вечных «половинчатых» вложений.
Методика памяти (`student-tests/upload_memory.py`): три сценария — 10 x PNG из случайного шума ~9,5 MiB (почти несжимаемый, максимум трафика); 10 x PNG 8000x5000 RGB (40 Мпикс, максимум распаковки); 20 смешанных. Перед каждым сценарием `docker compose restart api` (сбрасывает пик), прогрев 8 загрузками, затем все потоки стартуют одновременно (barrier) через балансировщик. Пик RSS — `VmHWM` из `/proc/1/status` процесса API (максимум за жизнь процесса по данным ядра, короткие всплески не теряются), дополнительно пик памяти cgroup контейнера и опрос раз в ~0,1–0,5 с. Проверяются коды 201 и SHA-256 скачанных файлов. Клиент — тот же компьютер, поэтому время загрузки ограничено и клиентом; это замер памяти, а не пропускной способности.
<!-- measurements:begin -->
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB.
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
| --- | --- | --- | --- | --- | --- |
| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 |
| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 |
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 |
| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 |
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 |
| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 |
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 |
| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 |
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет |
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет |
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
<!-- measurements:end -->
Вывод по памяти: рост RSS реплики при 10–20 одновременных загрузках — единицы MiB (см. таблицу), а не «размер файла x число загрузок»; верхняя оценка на загрузку — 256 KiB буфера временного файла + 64 KiB куска чтения + буферы boto3 при PUT; на проверку — 64 KiB распаковки PNG или ~2 MiB для JPEG, не больше 2 одновременно. Пик cgroup выше пика RSS, потому что в него входит page cache временных файлов (ядро освобождает его при нехватке памяти, OOM он не вызывает).
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Выдерживается отказ одной API-реплики и временная недоступность S3 (файловые операции 503, остальное работает). Не выдерживается отказ единственных `lb`, `postgres`, `redis`, `s3` и всей машины: стенд на одном узле.
- Повтор загрузки после потерянного ответа создаёт второе вложение (в контракте нет ключа идемпотентности для загрузки). Лишнее вложение, не прикреплённое к сообщению, остаётся: удаления в API нет; janitor удаляет только незавершённые загрузки.
- `missing_objects` janitor только находит; восстановление — из резервной копии S3, которой в стенде нет.
- Обход bucket janitor’ом держит в памяти список ключей (для учебного объёма — мегабайты); для миллионов объектов нужна сверка по диапазонам ключей или S3 Inventory.
- Временный файл загрузки (часть тела сверх 256 KiB) лежит на диске контейнера. В Linux `tempfile.TemporaryFile` создаётся уже без имени в каталоге (O_TMPFILE), поэтому даже при SIGKILL реплики место освобождается ядром, «забытых» файлов не бывает. Одновременно на диске может быть до 10 MiB на каждую идущую загрузку; лимита на число одновременных загрузок, кроме лимитов HAProxy/uvicorn, нет.
- Проверяется структура изображения, а не его безопасность для всех декодеров (например, JPEG с некорректными, но декодируемыми данными пройдёт). Перекодирование оригинала — задача lab5.
- MinIO собирался быть первым выбором, но его образы больше не доступны в публичных реестрах: закреплённый тег защищает от неожиданного обновления, но не от исчезновения образа. Для эксплуатации нужно зеркалить образы в свой реестр. SeaweedFS запущен одним процессом `weed server` на одном диске — это SPOF и не защита от потери диска; веб-интерфейсы master/filer без авторизации, закрыты только сетью.
- Скрипты `failover_metrics.py`, `rolling_update.py`, `test_expiry.py` из lab3 остались в `student-tests/` и работают со стендом, но в общий прогон lab4 не входят (это уровень 4 lab3, а не lab4).
+72
View File
@@ -0,0 +1,72 @@
"""Сохранение вложения: PostgreSQL (метаданные) + S3 (байты) без распределённой транзакции.
Порядок записи и что происходит при отказе на каждом шаге:
1. INSERT attachments (state=uploading) и COMMIT — упало: в S3 ничего нет, клиенту 503.
2. PUT объекта в S3 — упало: удаляем объект (на случай, если PUT
на самом деле прошёл) и строку, клиенту 503.
Если и это не вышло — остаётся uploading-строка,
её уберёт janitor после grace period.
3. UPDATE state=ready и COMMIT — упало/неизвестно: клиенту 503; строка остаётся
uploading, объект есть -> janitor удалит оба.
Объект сразу НЕ удаляем: при обрыве связи COMMIT
мог пройти, и удаление сломало бы ready-вложение.
4. 201 клиенту — только после шага 3.
Клиенту видны и прикрепляемы только ready-строки, поэтому незавершённая загрузка ни на что не влияет.
"""
import logging
from uuid import UUID, uuid4
import db
import storage
from errors import unavailable
from uploads import ImageInfo, Upload
log = logging.getLogger("messenger.attachments")
def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict:
attachment_id = uuid4()
key = storage.new_key(uuid4().hex)
db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type,
upload.size, upload.sha256, image.width, image.height, key)
try:
storage.put(key, upload.file, upload.size, image.content_type, upload.sha256)
except storage.StorageUnavailable:
compensate(attachment_id, key)
raise unavailable("File storage is temporarily unavailable") from None
row = db.finish_upload(attachment_id)
if row is None:
# janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace).
log.warning("Upload %s was reclaimed before it finished", attachment_id)
raise unavailable()
return row
def compensate(attachment_id: UUID, key: str) -> None:
"""Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor."""
try:
storage.delete(key, quick=True)
db.delete_upload_intent(attachment_id)
log.info("Upload %s rolled back after S3 failure", attachment_id)
except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту
log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__)
def public(row: dict) -> dict:
"""Строка БД -> Attachment из контракта (без object_key и размеров в пикселях)."""
return {
"id": row["id"],
"chat_id": row["chat_id"],
"owner_id": row["owner_id"],
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": "ready",
"variants": [],
"error_code": None,
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
+139
View File
@@ -0,0 +1,139 @@
import base64
import binascii
import hashlib
import logging
import secrets
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from uuid import UUID
import redis
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from redis.backoff import NoBackoff
from redis.retry import Retry
import settings
from errors import BASIC_CHALLENGE, unauthorized, unavailable
log = logging.getLogger("messenger.auth")
hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1)
DUMMY_HASH = hasher.hash("there-is-no-such-user")
PASSWORD_MIN_LENGTH = 12
PASSWORD_MAX_LENGTH = 128
def hash_password(password: str) -> str:
return hasher.hash(password)
def password_matches(password_hash: str | None, password: str) -> bool:
try:
hasher.verify(password_hash or DUMMY_HASH, password)
except (VerificationError, InvalidHashError):
return False
return password_hash is not None
def parse_basic(header: str | None) -> tuple[str, str] | None:
"""'Basic base64(username:password)' -> (username, password) или None.
Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617).
"""
scheme, _, value = (header or "").strip().partition(" ")
if scheme.lower() != "basic" or not value or len(value) > 1024:
return None
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError):
return None
username, colon, password = decoded.partition(":")
if not colon or not username or not username.isascii():
return None
return username, password
def parse_bearer(header: str | None) -> str | None:
scheme, _, value = (header or "").strip().partition(" ")
value = value.strip()
if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value:
return None
return value
KEY_PREFIX = "session:"
_redis = redis.Redis(
connection_pool=redis.BlockingConnectionPool.from_url(
settings.REDIS_URL or "redis://localhost:6379/0",
max_connections=settings.REDIS_POOL_MAX,
timeout=settings.REDIS_TIMEOUT_MS / 1000,
socket_timeout=settings.REDIS_TIMEOUT_MS / 1000,
socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000,
retry=Retry(NoBackoff(), 0),
decode_responses=True,
)
)
@dataclass(frozen=True)
class Session:
user_id: UUID
key: str
def session_key(token: str) -> str:
return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest()
def create_session(user_id: UUID) -> tuple[str, datetime]:
"""Новая сессия: случайный токен и абсолютное время истечения."""
token = secrets.token_urlsafe(32)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS)
try:
_redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000))
except redis.RedisError as exc:
log.warning("Redis is unavailable on login: %s", type(exc).__name__)
raise unavailable() from None
return token, expires_at
def check_bearer(header: str | None) -> Session:
token = parse_bearer(header)
if token is None:
raise unauthorized("Bearer token is required")
key = session_key(token)
try:
user_id = _redis.get(key)
except redis.RedisError as exc:
log.warning("Redis is unavailable: %s", type(exc).__name__)
raise unavailable() from None
if user_id is None:
raise unauthorized("Session is invalid or expired")
return Session(UUID(user_id), key)
def delete_session(session: Session) -> None:
try:
_redis.delete(session.key)
except redis.RedisError:
raise unavailable() from None
def ping_redis() -> None:
try:
_redis.ping()
except redis.RedisError:
raise unavailable() from None
def login_failed():
return unauthorized("Invalid username or password", BASIC_CHALLENGE)
def close() -> None:
_redis.close()
_redis.connection_pool.disconnect()
+211
View File
@@ -0,0 +1,211 @@
name: messenger-lab4
x-app-build: &app-build
context: .
args:
APP_VERSION: ${APP_VERSION:-dev}
x-s3-api-secrets: &s3-api-secrets
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
services:
migrate:
build: *app-build
command: ["python", "migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres: {condition: service_healthy}
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
# S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений.
# Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config
# (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket.
# Анонимного доступа нет: identity "anonymous" в конфиге не объявлена.
s3:
image: chrislusf/seaweedfs:3.80
entrypoint: ["/usr/bin/weed"]
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0",
"-master.volumeSizeLimitMB=1024", "-volume.max=0",
"-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"]
secrets: [s3_config]
volumes: ["s3data:/data"]
networks: [data]
healthcheck:
# S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт»)
test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"]
interval: 5s
timeout: 3s
retries: 24
start_period: 15s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
# Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки.
s3-init:
build: *app-build
command: ["python", "s3_init.py"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
s3: {condition: service_healthy}
healthcheck:
disable: true
restart: "no"
deploy:
resources:
limits: {cpus: "0.25", memory: 128M}
api:
build: *app-build
env_file: .env
environment:
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
<<: *s3-api-secrets
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
networks: [edge, data]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
redis: {condition: service_healthy}
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "1.00", memory: 512M}
# Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ.
janitor:
build: *app-build
command: ["python", "janitor.py"]
env_file: .env
environment:
CACHE_URL_FILE: "" # janitor не ходит в Redis
<<: *s3-api-secrets
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
healthcheck:
disable: true
restart: unless-stopped
stop_grace_period: 10s
deploy:
resources:
limits: {cpus: "0.25", memory: 192M}
lb:
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.1:8404:8404"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
networks: [edge]
depends_on:
api: {condition: service_healthy}
healthcheck:
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
interval: 5s
timeout: 3s
retries: 3
start_period: 10s
start_interval: 1s
restart: unless-stopped
stop_grace_period: 15s
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
postgres:
image: postgres:17.11-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"]
secrets: [postgres_password]
volumes: ["pgdata:/var/lib/postgresql/data"]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "1.00", memory: 512M}
redis:
image: redis:7.4.11-alpine
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
volumes:
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
- redisdata:/data
secrets: [redis_auth]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 12
restart: unless-stopped
deploy:
resources:
limits: {cpus: "0.50", memory: 128M}
networks:
edge: {}
data:
internal: true
volumes:
pgdata: {}
redisdata: {}
s3data: {}
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
redis_auth: {file: ./secrets/redis_auth}
s3_root_user: {file: ./secrets/s3_root_user}
s3_root_password: {file: ./secrets/s3_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
s3_config: {file: ./secrets/s3_config.json}
+359
View File
@@ -0,0 +1,359 @@
import logging
import time
from contextlib import contextmanager
from datetime import datetime
from uuid import UUID, uuid4
import psycopg
from psycopg import errors as pg_errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
import settings
from errors import bad_request, conflict, not_found, unavailable
log = logging.getLogger("messenger.db")
pool = ConnectionPool(
conninfo=settings.DATABASE_URL,
min_size=settings.DB_POOL_MIN,
max_size=settings.DB_POOL_MAX,
timeout=settings.DB_POOL_WAIT_SECONDS,
kwargs={
"connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS,
"row_factory": dict_row,
"options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}"
" -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC",
},
check=ConnectionPool.check_connection,
open=False,
)
@contextmanager
def connection(wait_seconds: float | None = None):
try:
with pool.connection(timeout=wait_seconds) as conn:
yield conn
except (psycopg.OperationalError, PoolTimeout) as exc:
log.warning("PostgreSQL is unavailable: %s", type(exc).__name__)
raise unavailable() from None
except psycopg.DataError as exc:
log.warning("PostgreSQL rejected a value: %s", type(exc).__name__)
raise bad_request("Invalid request data") from None
TEXT_FIELDS = ("display_name", "title", "text", "filename")
def to_db(value: str) -> bytes:
return value.encode("utf-8")
def from_db(row: dict | None) -> dict | None:
"""Строка из БД -> dict для API: байты текстовых полей обратно в str."""
if row is None:
return None
return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()}
def ping() -> None:
with connection(wait_seconds=1) as conn:
conn.execute("SELECT 1")
MIGRATIONS_LOCK = 20260202
def migration_files() -> list:
return sorted(settings.MIGRATIONS_DIR.glob("*.sql"))
def run_migrations() -> list[str]:
files = migration_files()
applied = []
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,))
try:
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now())""")
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
for path in files:
if path.name in done:
continue
with conn.transaction():
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
applied.append(path.name)
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,))
return applied
def wait_and_migrate(max_wait_seconds: int = 60) -> None:
deadline = time.monotonic() + max_wait_seconds
while True:
try:
applied = run_migrations()
log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)")
return
except psycopg.OperationalError:
if time.monotonic() > deadline:
raise
log.warning("PostgreSQL is not ready yet, retry in 1 s")
time.sleep(1)
def missing_migrations() -> list[str]:
expected = [path.name for path in migration_files()]
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
try:
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
except pg_errors.UndefinedTable:
return expected
return [name for name in expected if name not in done]
def wait_for_schema(max_wait_seconds: int) -> None:
deadline = time.monotonic() + max_wait_seconds
while True:
try:
missing = missing_migrations()
except psycopg.OperationalError:
missing = None
if missing == []:
log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files()))
return
if time.monotonic() > deadline:
raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}")
log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None
else "missing " + ", ".join(missing))
time.sleep(1)
USER_COLUMNS = "id, username, display_name, created_at"
def create_user(username: str, display_name: str, password_hash: str) -> dict:
try:
with connection() as conn:
return from_db(conn.execute(
f"""INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""",
(uuid4(), username, to_db(display_name), password_hash),
).fetchone())
except pg_errors.UniqueViolation:
raise conflict("Username already exists") from None
def get_user(user_id: UUID) -> dict | None:
with connection() as conn:
return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone())
def get_user_with_password(username: str) -> dict | None:
with connection() as conn:
return from_db(conn.execute(
f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,)
).fetchone())
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids"""
def create_chat(title: str, member_ids: list[UUID]) -> dict:
chat_id = uuid4()
members = sorted(member_ids)
try:
with connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
(chat_id, to_db(title)),
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat_id, member_id) for member_id in members],
)
except pg_errors.ForeignKeyViolation:
raise bad_request("Unknown member") from None
return {**from_db(row), "member_ids": members}
def get_chat(chat_id: UUID, user_id: UUID) -> dict:
with connection() as conn:
chat = conn.execute(
f"""SELECT {CHAT_COLUMNS} FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
WHERE c.id = %s""",
(user_id, chat_id),
).fetchone()
if chat is None:
raise not_found("Chat not found")
return from_db(chat)
def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
"""Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё»."""
where, params = "", [user_id]
if after is not None:
where = "WHERE (c.created_at, c.id) > (%s, %s)"
params += list(after)
with connection() as conn:
rows = conn.execute(
f"""SELECT {CHAT_COLUMNS} FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
{where}
ORDER BY c.created_at, c.id
LIMIT %s""",
(*params, limit + 1),
).fetchall()
return [from_db(row) for row in rows]
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids"""
def is_member(conn, chat_id: UUID, user_id: UUID) -> bool:
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
).fetchone() is not None
def check_member(chat_id: UUID, user_id: UUID) -> None:
with connection() as conn:
if not is_member(conn, chat_id, user_id):
raise not_found("Chat not found")
def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID],
idempotency_key: str, request_hash: bytes) -> dict:
try:
with connection() as conn:
with conn.transaction():
if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404
raise not_found("Chat not found")
if attachment_ids:
# Только готовые вложения ЭТОГО чата. FOR SHARE: строку не удалят до конца транзакции.
found = conn.execute(
"""SELECT id FROM attachments
WHERE id = ANY(%s) AND chat_id = %s AND state = 'ready' FOR SHARE""",
(attachment_ids, chat_id),
).fetchall()
if len(found) != len(attachment_ids):
raise not_found("Attachment not found")
created = conn.execute(
"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING
RETURNING id, chat_id, sender_id, text, created_at""",
(uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash),
).fetchone()
if created is not None:
if attachment_ids:
with conn.cursor() as cur:
cur.executemany(
"""INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
VALUES (%s, %s, %s, %s)""",
[(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)],
)
return {**from_db(created), "attachment_ids": list(attachment_ids)}
saved = conn.execute(
f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m
WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""",
(chat_id, sender_id, idempotency_key),
).fetchone()
except pg_errors.ForeignKeyViolation:
raise not_found("Chat not found") from None
if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен
raise unavailable()
if bytes(saved.pop("request_hash")) != request_hash:
raise conflict("Idempotency-Key was already used with a different request")
return from_db(saved)
def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
"""История чата после курсора (keyset-пагинация), до limit+1 строк."""
where, params = "m.chat_id = %s", [chat_id]
if after is not None:
where += " AND (m.created_at, m.id) > (%s, %s)"
params += list(after)
with connection() as conn:
if not is_member(conn, chat_id, user_id):
raise not_found("Chat not found")
rows = conn.execute(
f"""SELECT {MESSAGE_COLUMNS} FROM messages m
WHERE {where}
ORDER BY m.created_at, m.id
LIMIT %s""",
(*params, limit + 1),
).fetchall()
return [from_db(row) for row in rows]
# ---------- вложения ----------
ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"created_at, updated_at, object_key")
def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str,
size_bytes: int, sha256: str, width: int, height: int, object_key: str) -> None:
"""Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в S3.
Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт
и удалит его, если загрузка не дошла до ready (обрыв, падение реплики, отказ S3 или БД).
"""
try:
with connection() as conn:
conn.execute(
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
width, height, object_key, state)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""",
(attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256,
width, height, object_key),
)
except pg_errors.ForeignKeyViolation:
raise not_found("Chat not found") from None
def finish_upload(attachment_id: UUID) -> dict | None:
"""Шаг 3 из 3: объект в S3 уже сохранён -> ready. None, если строку успел убрать janitor."""
with connection() as conn:
row = conn.execute(
f"""UPDATE attachments SET state = 'ready', updated_at = now()
WHERE id = %s AND state = 'uploading'
RETURNING {ATTACHMENT_COLUMNS}""",
(attachment_id,),
).fetchone()
return from_db(row)
def delete_upload_intent(attachment_id: UUID) -> None:
with connection(wait_seconds=1) as conn:
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,))
def get_attachment(attachment_id: UUID, user_id: UUID) -> dict:
"""Готовое вложение, если user_id — участник его чата. Иначе 404 (не раскрываем существование)."""
with connection() as conn:
row = conn.execute(
f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))}
FROM attachments a
JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s
WHERE a.id = %s AND a.state = 'ready'""",
(user_id, attachment_id),
).fetchone()
if row is None:
raise not_found("Attachment not found")
return from_db(row)
+40
View File
@@ -0,0 +1,40 @@
# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт.
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions
BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы
class ApiError(Exception):
def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None):
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers or {}
def bad_request(message: str) -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError:
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
def not_found(message: str) -> ApiError:
return ApiError(404, "not_found", message)
def conflict(message: str) -> ApiError:
return ApiError(409, "conflict", message)
def unavailable(message: str = "Service temporarily unavailable") -> ApiError:
return ApiError(503, "unavailable", message)
def payload_too_large(message: str) -> ApiError:
return ApiError(413, "payload_too_large", message)
def unsupported_media_type(message: str) -> ApiError:
return ApiError(415, "unsupported_media_type", message)
+67
View File
@@ -0,0 +1,67 @@
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
# date: 2026-10-21 21:03:05, commit: 2dbed80
# command: python.exe student-tests/test_attachments.py
# exit code: 0, 4.0 s
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
exactly 10485760 bytes (10 MiB) -> 201
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
PASS: the 10 MiB limit is exact
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
PASS: oversized streamed body is cut off, the API keeps working
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 201 5ffe6847
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
PASS: pixel limit is checked before decompression
PNG cut in half (55908 bytes, image/png) -> 415
PNG without IEND (111805 bytes, image/png) -> 415
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
PNG with garbage after IEND (111824 bytes, image/png) -> 415
PNG signature only (8 bytes, image/png) -> 415
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
JPEG cut in half (746 bytes, image/jpeg) -> 415
JPEG header only (20 bytes, image/jpeg) -> 415
random bytes (1600 bytes, image/png) -> 415
plain text (28 bytes, image/jpeg) -> 415
empty body -> 400
PASS: damaged and foreign files are rejected with 415, never 500
PNG bytes sent as image/jpeg -> 415
JPEG bytes sent as image/png -> 415
GIF bytes sent as image/png -> 415
PNG sent as image/gif -> 415
PNG sent as text/html -> 415
PNG sent as application/octet-stream -> 415
PNG sent as multipart/form-data -> 415
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
PASS: the type is taken from the content; a declared type that does not match is refused
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
no filename -> 400
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
PASS: filename never becomes a path; dangerous names are refused
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
upload into someone else's / unknown chat -> 404, without token -> 401
variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200
PASS: only chat members can see an attachment
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
attachment of another chat (author is a member of both) -> 404
own + foreign attachment -> 404
unknown attachment_id -> 404
outsider referencing an attachment of the chat -> 404
duplicate ids -> 400
11 ids -> 400
bad UUID -> 400
empty text without attachments -> 400
whitespace text without attachments -> 400
PASS: only attachments of the same chat can be linked; rejected requests change nothing
same key + same attachments -> same 201; other attachments -> 409; missing field == []
PASS: Idempotency-Key covers attachment_ids
sent 1.1 MB of a declared 5 MB body and closed the connection
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
PASS: an interrupted upload is dropped
11 passed, 0 failed
+18
View File
@@ -0,0 +1,18 @@
# balancing: balancing, shared sessions/data/files, X-Instance-Id
# date: 2026-10-21 21:08:50, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/test_balancing.py
# exit code: 0, 1.9 s
20 x GET /users/me with one token -> replicas: {'6128eb268c44': 10, '11a93acecaca': 10}
PASS: one session works on every replica, no second login (sessions are shared in Redis)
message written by 11a93acecaca; read back by ['11a93acecaca', '6128eb268c44']
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
logout on 6128eb268c44; old token then rejected by ['11a93acecaca', '6128eb268c44']
PASS: logout on one replica revokes the session on all of them
first POST on 6128eb268c44 -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af; same key on 11a93acecaca -> 201 296cb1bf-8c1c-46f2-942a-4865d1b7d6af
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
file uploaded via 6128eb268c44; downloaded byte for byte via ['11a93acecaca', '6128eb268c44']
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['11a93acecaca', '6128eb268c44']; api containers: ['11a93acecaca', '6128eb268c44']
PASS: X-Instance-Id is in every response and points to a real container
6 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# check: offline contract check (make check)
# date: 2026-10-21 21:02:57, commit: 2dbed80
# command: python.exe scripts/check_contract.py
# exit code: 0, 0.1 s
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+47
View File
@@ -0,0 +1,47 @@
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
# date: 2026-10-21 21:08:14, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/test_consistency.py
# exit code: 0, 148.7 s
declared 9 000 000 bytes, sent 211817, closed the connection
declared 9 000 000 bytes, sent 3111817, closed the connection
rows in attachments: 32 -> 32; objects in bucket: 31 -> 31
temporary upload files in api containers: {'11a93acecaca': 0, '6128eb268c44': 0} (removed when the request ends)
api containers have no volumes: the original is never stored locally
PASS: an interrupted upload leaves no row, no object and no local file
crash after S3 PUT: row 04aacbf6 state=uploading, object originals/219175ec1d5647f0aeaec84ee8243cb3 exists in S3
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.101} -> fresh row and object are kept (the upload might still be running)
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "objects_scanned": 33, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.092} -> nothing deleted
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.096} -> row and object deleted
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 32, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.103} -> idempotent
the ready attachment of the same chat is untouched
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
object originals/867f5c5d42bf4bf9973b60ef8856d88f put into the bucket without a row in PostgreSQL
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.102} -> fresh object kept
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.098} -> reported, kept
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "objects_scanned": 34, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.1} -> orphan deleted
ready attachments older than the grace period are untouched (they have rows)
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500)
PASS: orphans are found and deleted only after the grace period; missing objects are reported
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
upload -> 503 unavailable in 20141 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose start s3
after S3 recovery the old file downloads byte for byte
new upload -> 201
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 35, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.099}
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
--- S3 frozen (no answer, timeouts): docker compose pause s3
upload -> 503 unavailable in 14766 ms, no storage details in the body
download of an existing file -> 503
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
docker compose unpause s3
after S3 recovery the old file downloads byte for byte
new upload -> 201
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "objects_scanned": 37, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.094}
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
5 passed, 0 failed
+21
View File
@@ -0,0 +1,21 @@
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
# date: 2026-10-21 21:09:10, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/test_db.py
# exit code: 0, 14.2 s
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 57 -> 58
PASS: chat row was inserted and then rolled back together with members
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
PASS: the database itself rejects a message from a non-member
stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97)
PASS: password is stored as Argon2id hash
unique constraint on users: UNIQUE (username)
PASS: username is unique in the database
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; running the migrate service once more
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql; log: Migrations applied: none (schema is up to date)
PASS: migrations are applied once, a repeated run does nothing
api replicas that checked the schema at start: 8; api log lines 'Migrations applied': 0; migrate service: Exited (0) 6 minutes ago
PASS: only the migrate service applies migrations, replicas wait for the schema
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
7 passed, 0 failed
+99
View File
@@ -0,0 +1,99 @@
=== Data volume
messages_total | messages_in_big_chat
----------------+----------------------
20146 | 15000
=== History index
indexdef
----------------------------------------------------------------------------------------
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
QUERY PLAN
------------------------------------------------------------------------------------------------
Limit (actual time=0.005..0.011 rows=51 loops=1)
Buffers: shared hit=5
-> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1)
Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid)
Buffers: shared hit=5
Planning:
Buffers: shared hit=6
Planning Time: 0.060 ms
Execution Time: 0.020 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.005..0.012 rows=51 loops=1)
Buffers: shared hit=6
-> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1)
Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid)))
Buffers: shared hit=6
Planning Time: 0.044 ms
Execution Time: 0.031 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=0.020..0.225 rows=51 loops=1)
Buffers: shared hit=60
-> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1)
Index Cond: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid)))
Buffers: shared hit=60
SubPlan 1
-> Sort (actual time=0.003..0.003 rows=0 loops=51)
Sort Key: a."position"
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
Recheck Cond: (message_id = m.id)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Index Cond: (message_id = m.id)
Buffers: shared hit=51
Planning:
Buffers: shared hit=63
Planning Time: 0.157 ms
Execution Time: 0.242 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
QUERY PLAN
---------------------------------------------------------------------------------------------------
Limit (actual time=0.878..0.884 rows=51 loops=1)
Buffers: shared hit=258
-> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1)
Index Cond: (chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid)
Buffers: shared hit=258
Planning Time: 0.037 ms
Execution Time: 0.897 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
QUERY PLAN
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Limit (actual time=1.097..1.101 rows=51 loops=1)
Buffers: shared hit=261
-> Sort (actual time=1.097..1.098 rows=51 loops=1)
Sort Key: created_at, id
Sort Method: top-N heapsort Memory: 31kB
Buffers: shared hit=261
-> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1)
Filter: ((chat_id = '7c307498-c398-4a1e-be96-13556a559cbf'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '170ea586-3f7e-4314-a9bf-8e4f394941ac'::uuid)))
Rows Removed by Filter: 15146
Buffers: shared hit=261
Planning:
Buffers: shared hit=4
Planning Time: 0.051 ms
Execution Time: 1.115 ms
=== ROLLBACK: test data removed, index is back
explain_users_left
--------------------
0
+43
View File
@@ -0,0 +1,43 @@
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
# date: 2026-10-21 21:09:14, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/explain.py
# exit code: 0, 1.1 s
=== Data volume
=== History index
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
Buffers: shared hit=5
-> Index Scan using messages_history on messages (actual time=0.005..0.008 rows=51 loops=1)
Buffers: shared hit=5
Buffers: shared hit=6
Execution Time: 0.020 ms
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
Buffers: shared hit=6
-> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1)
Buffers: shared hit=6
Execution Time: 0.031 ms
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
Buffers: shared hit=60
-> Index Scan using messages_history on messages m (actual time=0.019..0.222 rows=51 loops=1)
Buffers: shared hit=60
Buffers: shared hit=54
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
Buffers: shared hit=51
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
Buffers: shared hit=51
Buffers: shared hit=63
Execution Time: 0.242 ms
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
Buffers: shared hit=258
-> Index Scan using messages_history on messages (actual time=0.007..0.669 rows=10051 loops=1)
Buffers: shared hit=258
Execution Time: 0.897 ms
=== 4. Query 2 without the index (index is dropped only inside this transaction)
Buffers: shared hit=261
Buffers: shared hit=261
-> Seq Scan on messages (actual time=0.307..0.737 rows=5000 loops=1)
Buffers: shared hit=261
Buffers: shared hit=4
Execution Time: 1.115 ms
=== ROLLBACK: test data removed, index is back
PASS: full plans saved to evidence\explain-history.txt
+33
View File
@@ -0,0 +1,33 @@
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
# date: 2026-10-21 21:09:54, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/run_failover.py
# exit code: 0, 39.7 s
hook: scripts\stop-one.cmd; api replicas: ['11a93acecaca', '6128eb268c44']
=== run 1: api containers before
SERVICE CONTAINER ID STATUS
api 11a93acecaca Up 36 seconds (healthy)
api 6128eb268c44 Up 36 seconds (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.39}␍
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
SERVICE CONTAINER ID STATUS
api 11a93acecaca Exited (0) 11 seconds ago
api 6128eb268c44 Up 53 seconds (healthy)
the hook stopped 11a93acecaca (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
restored: docker start 11a93acecaca -> healthy and back in HAProxy
=== run 2: api containers before
SERVICE CONTAINER ID STATUS
api 11a93acecaca Up 2 seconds (healthy)
api 6128eb268c44 Up 56 seconds (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 4.328}␍
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
api containers after
SERVICE CONTAINER ID STATUS
api 11a93acecaca Up 18 seconds (healthy)
api 6128eb268c44 Exited (0) 11 seconds ago
the hook stopped 6128eb268c44 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
restored: docker start 6128eb268c44 -> healthy and back in HAProxy
PASS: public failover test passed 2 times; every api replica was stopped once and restored
+29
View File
@@ -0,0 +1,29 @@
# failures: Redis and PostgreSQL outages via the balancer
# date: 2026-10-21 21:11:20, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/test_failures.py
# exit code: 0, 55.6 s
stopping redis
/health/live -> 200, /health/ready -> 503 (8016 ms)
valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked
unknown token -> 503, not 200: nobody is trusted without a check
no token -> 401
login -> 503: there is nowhere to store the session
registration -> 201: it needs only PostgreSQL
starting redis
after start: the old token works, confirmed messages are in place
PASS: Redis outage gives 503 and does not let anyone in without a session check
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
right after the crash: GET /users/me -> 503
starting redis
after start: the session issued before the crash still works (AOF, appendfsync always)
PASS: sessions survive a Redis crash, not only a clean restart
stopping postgres
/health/live -> 200, /health/ready -> 503 (1016 ms)
valid token: users/me, chats, history, new message -> 503 unavailable, no details
no token -> 401, unknown token -> 401: anonymous users are not let in
login -> 503, registration -> 503
starting postgres
after start: the old token works, confirmed messages are in place
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
3 passed, 0 failed
+22
View File
@@ -0,0 +1,22 @@
# idempotency: Idempotency-Key, parallel same key via replicas
# date: 2026-10-21 21:08:52, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/test_idempotency.py
# exit code: 0, 2.4 s
first -> 201; same JSON written differently -> 201; plain retry -> 201
PASS: a retry with the same key returns the original 201 and message, no duplicate
20 parallel POSTs with one key: statuses {201: 20}, replicas {'6128eb268c44': 10, '11a93acecaca': 10}, different bodies in answers: 1, messages in history: 1
PASS: parallel requests with one key through different replicas create exactly one message
20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A']
PASS: the first body wins, the same body gets 201, the other body gets 409
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
key used by a 400 and by a 404 first, then valid request -> 201
PASS: rejected requests (400, 404) do not store a result for the key
same key and body by an outsider -> 404; by alice after logout -> 401
PASS: a retry checks the session and chat access again, the saved answer is not leaked
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
PASS: an unconfirmed POST can be safely retried with the same key
8 passed, 0 failed
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 4,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.015,
"concurrency": 4,
"successful_requests": 7379,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 245.844,
"successful_latency_ms": {
"p50": 16.0,
"p95": 32.0,
"p99": 32.0
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+22
View File
@@ -0,0 +1,22 @@
# load: public load generator, 30 s, concurrency 4
# date: 2026-10-21 21:10:24, commit: 2dbed80 + uncommitted changes
# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
# exit code: 0, 30.4 s
{
"lab": 4,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.015,
"concurrency": 4,
"successful_requests": 7379,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 245.844,
"successful_latency_ms": {
"p50": 16.0,
"p95": 32.0,
"p99": 32.0
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+24
View File
@@ -0,0 +1,24 @@
# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence)
# date: 2026-10-21 21:12:49, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/run_persistence.py
# exit code: 0, 40.2 s
before: container, ID, started at
messenger-lab4-api-1 11a93acecaca 2026-10-21T18:11:37.067348364Z
messenger-lab4-api-2 6128eb268c44 2026-10-21T18:11:36.916459116Z
messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:11:33.313531682Z
messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:11:40.981519931Z
messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:11:22.482229396Z
messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:11:22.626999402Z
messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:11:32.264154477Z
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.␍
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
after (hook + waiting for readiness took 39.2 s): container, ID, started at, restarted?
messenger-lab4-api-1 11a93acecaca 2026-10-21T18:12:46.288933033Z same container: True, restarted: True
messenger-lab4-api-2 6128eb268c44 2026-10-21T18:12:46.174283308Z same container: True, restarted: True
messenger-lab4-janitor-1 baefc0e97dff 2026-10-21T18:12:42.503993973Z same container: True, restarted: True
messenger-lab4-lb-1 03697e5fe34c 2026-10-21T18:12:47.684809415Z same container: True, restarted: True
messenger-lab4-postgres-1 f7a6a1ce25a7 2026-10-21T18:12:11.527270274Z same container: True, restarted: True
messenger-lab4-redis-1 b4523b39a0fe 2026-10-21T18:12:11.368416107Z same container: True, restarted: True
messenger-lab4-s3-1 e7cbe6ddfa4a 2026-10-21T18:12:21.264135454Z same container: True, restarted: True
PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, new start time); data, the attachment bytes and the old token survived
+10
View File
@@ -0,0 +1,10 @@
# race: lab2 races via the balancer
# date: 2026-10-21 21:08:56, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/test_race.py
# exit code: 0, 1.2 s
sent 100 messages from 10 clients in 0.22 s; read 100 in 4 pages, unique ids: 100
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
20 parallel registrations of one username: 201 x1, 409 x19
PASS: concurrent registration created exactly one user
2 passed, 0 failed
+14
View File
@@ -0,0 +1,14 @@
# redis: Redis as the session store
# date: 2026-10-21 21:09:13, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/redis_info.py
# exit code: 0, 2.8 s
session key: session:f4f1fe0235d139de...; value is the user id: True
TTL of the key, seconds: 3599
sessions in Redis: 126
config appendonly = yes
config appendfsync = always
config maxmemory-policy = noeviction
aof_enabled:1
aof_last_write_status:ok
without password: NOAUTH Authentication required.
+44
View File
@@ -0,0 +1,44 @@
# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap
# date: 2026-10-21 21:03:14, commit: 2dbed80
# command: python.exe student-tests/test_s3_security.py
# exit code: 0, 9.5 s
anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied
anonymous head_object directly to S3 (from the data network) -> 403
anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied
anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied
anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied
the same file through the API for a chat member -> 200
PASS: the bucket is private: no anonymous read, list, write or policy read
API key: list buckets -> ['messenger-attachments']
API key: get own object -> allowed
API key: put object -> allowed
API key: delete object -> allowed
API key: list bucket -> allowed
API key: create another bucket -> denied: AccessDenied
API key: make bucket public (put policy) -> denied: NotImplemented
API key: read bucket policy -> denied: NoSuchBucketPolicy
API key: delete bucket -> denied: AccessDenied
API key: change versioning -> denied: NotImplemented
PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied
compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand)
127.0.0.1:8333 from the host -> connection refused
127.0.0.1:9333 from the host -> connection refused
127.0.0.1:8888 from the host -> connection refused
127.0.0.1:19333 from the host -> connection refused
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of lb: -
secrets of migrate: ['database_url']
secrets of postgres: ['postgres_password']
secrets of redis: ['redis_auth']
secrets of s3: ['s3_config']
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init
s3-init: s3-init: bucket messenger-attachments exists
s3-init: s3-init: api key can put/get/list/delete objects of its bucket
s3-init: s3-init: api key cannot create buckets (AccessDenied)
s3-init: s3-init: bucket is private (anonymous list/get/put -> [403])
after the repeated bootstrap: old file downloads, new upload works
PASS: bucket/user/policy bootstrap can be repeated safely
4 passed, 0 failed
+6
View File
@@ -0,0 +1,6 @@
# smoke: public smoke via the balancer (make test), with attachment
# date: 2026-10-21 21:03:01, commit: 2dbed80
# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080
# exit code: 0, 0.6 s
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+40
View File
@@ -0,0 +1,40 @@
# stack: stand layout: replicas, ports, limits, networks, secrets, S3
# date: 2026-10-21 21:03:00, commit: 2dbed80
# command: python.exe student-tests/check_stack.py
# exit code: 0, 3.0 s
host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1
Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop
SERVICE CONTAINER ID STATUS PORTS
api 11a93acecaca Up 20 seconds (healthy) 8080/tcp
api 6128eb268c44 Up 20 seconds (healthy) 8080/tcp
janitor baefc0e97dff Up 20 seconds
lb 03697e5fe34c Up 18 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
migrate aca090330db4 Exited (0) 22 seconds ago
postgres f7a6a1ce25a7 Up 2 minutes (healthy)
redis b4523b39a0fe Up 2 minutes (healthy)
s3 e7cbe6ddfa4a Up 2 minutes (healthy)
s3-init 60719e674e18 Exited (0) 21 seconds ago
container image cpus memory host ports networks
messenger-lab4-api-1 messenger-lab4-api 1.00 512 MiB - data,edge
messenger-lab4-api-2 messenger-lab4-api 1.00 512 MiB - data,edge
messenger-lab4-janitor-1 messenger-lab4-janitor 0.25 192 MiB - data
messenger-lab4-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge
messenger-lab4-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data
messenger-lab4-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data
messenger-lab4-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
secrets of lb: -
secrets of migrate: ['database_url']
secrets of postgres: ['postgres_password']
secrets of redis: ['redis_auth']
secrets of s3: ['s3_config']
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
through http://127.0.0.1:8080 one session is served by 2 replicas: ['11a93acecaca', '6128eb268c44']
/health/ready -> 200 from 11a93acecaca
PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor only in the internal data network, S3 admin keys only in s3 and s3-init
+26
View File
@@ -0,0 +1,26 @@
# student-tests: lab1/lab2 API tests via the balancer
# date: 2026-10-21 21:08:55, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/test_api.py
# exit code: 0, 2.7 s
PASS: duplicate username returns 409
PASS: invalid JSON returns 400
PASS: wrong and extra fields return 400
PASS: foreign chat returns 404
PASS: empty and too long messages return 400, 4000 chars is ok
PASS: pagination works across multiple pages, foreign cursor returns 400
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
PASS: chat list pagination
PASS: unicode text is stored as is
PASS: strings with U+0000 are stored and returned as is
PASS: user response has no password or hash
PASS: password length 12..128, blank password is rejected
PASS: Basic login returns opaque Bearer token with expiry
PASS: wrong password and unknown user both return the same 401
PASS: malformed Basic header returns 401
PASS: password with colon and unicode works
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
PASS: X-User-Id does not change the current user
PASS: 401 comes before 400 and 404
PASS: logout revokes only the current session
20 passed, 0 failed
+40
View File
@@ -0,0 +1,40 @@
# Сводка проверок
| Шаг | Что проверяет | Результат | Время, с | Файл |
| --- | --- | --- | --- | --- |
| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 40 | evidence/persistence.txt |
# Замеры
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-21, commit 2dbed80 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB.
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
| --- | --- | --- | --- | --- | --- |
| 1 | 11a93acecaca | 2 | 126 | 0 | 4.39 |
| 2 | 6128eb268c44 | 2 | 124 | 0 | 4.328 |
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.38 / 1.52 | 11a93acecaca: 5 | 93.5 | 106.2 | 12.6 | 134.1 |
| | | | | 6128eb268c44: 5 | 94.8 | 105.3 | 10.5 | 135.2 |
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.85 / 1.09 | 11a93acecaca: 5 | 93.2 | 96.6 | 3.4 | 84.3 |
| | | | | 6128eb268c44: 5 | 93.1 | 94.7 | 1.6 | 86.8 |
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.37 / 1.77 | 11a93acecaca: 10 | 94.8 | 121.7 | 26.9 | 181.9 |
| | | | | 6128eb268c44: 10 | 95.0 | 116.1 | 21.0 | 155.6 |
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 245.844 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.39, 4.328 с (hook ждёт остановки ~4 с); потерь нет |
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 39.2 с до готовности; потерь нет |
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20141 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14766 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
+124
View File
@@ -0,0 +1,124 @@
{
"date": "2026-10-21 21:08",
"api_limit_mib": 512,
"api_limit_cpus": 1.0,
"runs": [
{
"scenario": "10 x ~9.5 MiB PNG (noise)",
"concurrency": 10,
"file_bytes": [
9953047
],
"total_mib": 94.9,
"all_201": true,
"wall_seconds": 1.52,
"latency_s": {
"p50": 1.38,
"max": 1.52
},
"sampler_rounds": 9,
"replicas": [
{
"replica": "11a93acecaca",
"uploads_served": 5,
"rss_before_mib": 93.5,
"rss_peak_mib": 106.2,
"rss_growth_mib": 12.6,
"rss_peak_sampled_mib": 106.2,
"cgroup_peak_mib": 134.1,
"cgroup_peak_sampled_mib": 127.7
},
{
"replica": "6128eb268c44",
"uploads_served": 5,
"rss_before_mib": 94.8,
"rss_peak_mib": 105.3,
"rss_growth_mib": 10.5,
"rss_peak_sampled_mib": 105.3,
"cgroup_peak_mib": 135.2,
"cgroup_peak_sampled_mib": 131.4
}
]
},
{
"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)",
"concurrency": 10,
"file_bytes": [
116714
],
"total_mib": 1.1,
"all_201": true,
"wall_seconds": 1.09,
"latency_s": {
"p50": 0.85,
"max": 1.09
},
"sampler_rounds": 8,
"replicas": [
{
"replica": "11a93acecaca",
"uploads_served": 5,
"rss_before_mib": 93.2,
"rss_peak_mib": 96.6,
"rss_growth_mib": 3.4,
"rss_peak_sampled_mib": 96.6,
"cgroup_peak_mib": 84.3,
"cgroup_peak_sampled_mib": 77.3
},
{
"replica": "6128eb268c44",
"uploads_served": 5,
"rss_before_mib": 93.1,
"rss_peak_mib": 94.7,
"rss_growth_mib": 1.6,
"rss_peak_sampled_mib": 94.6,
"cgroup_peak_mib": 86.8,
"cgroup_peak_sampled_mib": 77.3
}
]
},
{
"scenario": "20 mixed: large + 40 Mpx + JPEG",
"concurrency": 20,
"file_bytes": [
1492,
116714,
9953047
],
"total_mib": 133.2,
"all_201": true,
"wall_seconds": 1.77,
"latency_s": {
"p50": 1.37,
"max": 1.77
},
"sampler_rounds": 10,
"replicas": [
{
"replica": "11a93acecaca",
"uploads_served": 10,
"rss_before_mib": 94.8,
"rss_peak_mib": 121.7,
"rss_growth_mib": 26.9,
"rss_peak_sampled_mib": 121.7,
"cgroup_peak_mib": 181.9,
"cgroup_peak_sampled_mib": 180.6
},
{
"replica": "6128eb268c44",
"uploads_served": 10,
"rss_before_mib": 95.0,
"rss_peak_mib": 116.1,
"rss_growth_mib": 21.0,
"rss_peak_sampled_mib": 116.1,
"cgroup_peak_mib": 155.6,
"cgroup_peak_sampled_mib": 155.6
}
]
}
],
"single_file_extra_rss_mib": {
"naive": 161,
"streaming": 0
}
}
+13
View File
@@ -0,0 +1,13 @@
# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas
# date: 2026-10-21 21:08:48, commit: 2dbed80 + uncommitted changes
# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json
# exit code: 0, 34.2 s
client: Python 3.11.1 on Windows 10
preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...
large: 9953047 bytes; 40 Mpx: 116714 bytes
{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_201": true, "wall_seconds": 1.52, "latency_s": {"p50": 1.38, "max": 1.52}, "sampler_rounds": 9, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.5, "rss_peak_mib": 106.2, "rss_growth_mib": 12.6, "rss_peak_sampled_mib": 106.2, "cgroup_peak_mib": 134.1, "cgroup_peak_sampled_mib": 127.7}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 94.8, "rss_peak_mib": 105.3, "rss_growth_mib": 10.5, "rss_peak_sampled_mib": 105.3, "cgroup_peak_mib": 135.2, "cgroup_peak_sampled_mib": 131.4}]}
{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_201": true, "wall_seconds": 1.09, "latency_s": {"p50": 0.85, "max": 1.09}, "sampler_rounds": 8, "replicas": [{"replica": "11a93acecaca", "uploads_served": 5, "rss_before_mib": 93.2, "rss_peak_mib": 96.6, "rss_growth_mib": 3.4, "rss_peak_sampled_mib": 96.6, "cgroup_peak_mib": 84.3, "cgroup_peak_sampled_mib": 77.3}, {"replica": "6128eb268c44", "uploads_served": 5, "rss_before_mib": 93.1, "rss_peak_mib": 94.7, "rss_growth_mib": 1.6, "rss_peak_sampled_mib": 94.6, "cgroup_peak_mib": 86.8, "cgroup_peak_sampled_mib": 77.3}]}
{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_201": true, "wall_seconds": 1.77, "latency_s": {"p50": 1.37, "max": 1.77}, "sampler_rounds": 10, "replicas": [{"replica": "11a93acecaca", "uploads_served": 10, "rss_before_mib": 94.8, "rss_peak_mib": 121.7, "rss_growth_mib": 26.9, "rss_peak_sampled_mib": 121.7, "cgroup_peak_mib": 181.9, "cgroup_peak_sampled_mib": 180.6}, {"replica": "6128eb268c44", "uploads_served": 10, "rss_before_mib": 95.0, "rss_peak_mib": 116.1, "rss_growth_mib": 21.0, "rss_peak_sampled_mib": 116.1, "cgroup_peak_mib": 155.6, "cgroup_peak_sampled_mib": 155.6}]}
one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB
PASS: all uploads succeeded; largest RSS growth of one replica: 26.9 MiB (limit 512 MiB); results in evidence/upload-memory.json
+77
View File
@@ -0,0 +1,77 @@
# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики).
# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить
# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky).
global
log stdout format raw local0 info
maxconn 4096
# При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с.
hard-stop-after 10s
# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик.
# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска.
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s # перечитывать список реплик раз в 2 с
hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с
hold nx 2s
hold other 2s
hold refused 2s
hold timeout 2s
accepted_payload_size 8192
defaults
mode http
log global
option httplog
option dontlog-normal # в лог только ошибки, а не каждый запрос
timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос
timeout client 30s
# Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка
# при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт,
# сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT).
timeout server 30s
timeout http-request 10s
timeout http-keep-alive 5s
timeout queue 5s
timeout check 1s
# Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода.
# Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key.
retries 2
option redispatch 1
frontend api_in
bind :8080
option forwardfor
default_backend api
backend api
balance roundrobin
# Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis.
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
http-check expect status 200
# До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT.
# Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP.
# observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки.
server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none
# Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis
# или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате
# контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает.
# Пока готова хоть одна реплика, эти записи не используются.
# allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend).
server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
option allbackups
# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1).
# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда.
frontend stats
bind :8404
monitor-uri /lb-health
monitor fail if { nbsrv(api) lt 1 }
stats enable
stats uri /stats
stats refresh 2s
+13
View File
@@ -0,0 +1,13 @@
bind 0.0.0.0
port 6379
protected-mode yes
dir /data
appendonly yes
appendfsync always
save ""
maxmemory 64mb
maxmemory-policy noeviction
include /run/secrets/redis_auth
+57
View File
@@ -0,0 +1,57 @@
#!/bin/sh
# Bootstrap of the S3 storage (MinIO), run by the one-shot service s3-init before api starts.
# Idempotent: every step can be repeated (docker compose run --rm s3-init) without losing data.
# 1. private bucket S3_BUCKET (no anonymous access, versioning off);
# 2. policy messenger-api: only Get/Put/Delete/List objects of this bucket (JSON below);
# 3. separate user for api/janitor with this policy. Root credentials stay in s3 and s3-init only.
# Secrets are read from /run/secrets and are never printed.
set -eu
BUCKET="${S3_BUCKET:?S3_BUCKET is not set}"
ROOT_USER="$(cat /run/secrets/s3_root_user)"
ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
API_KEY="$(cat /run/secrets/s3_access_key_id)"
API_SECRET="$(cat /run/secrets/s3_secret_access_key)"
POLICY=messenger-api
export MC_CONFIG_DIR=/tmp/mc
# Least privilege for the api key: objects of ONE bucket. No bucket admin, policies, users or other buckets.
cat > /tmp/policy.json <<POLICY_JSON
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::${BUCKET}"]},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload"],
"Resource": ["arn:aws:s3:::${BUCKET}/*"]}
]
}
POLICY_JSON
i=0
until mc alias set local "${S3_ENDPOINT:-http://s3:9000}" "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do
i=$((i + 1))
[ "$i" -ge 30 ] && { echo "s3-init: MinIO is not reachable"; exit 1; }
sleep 1
done
mc mb --ignore-existing "local/${BUCKET}" >/dev/null
mc anonymous set none "local/${BUCKET}" >/dev/null
echo "s3-init: bucket ${BUCKET} exists"
mc admin policy create local "$POLICY" /tmp/policy.json >/dev/null
echo "s3-init: policy ${POLICY} created/updated"
mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null
info="$(mc admin user info local "$API_KEY" 2>/dev/null || true)"
case "$info" in
*"$POLICY"*) ;;
*) mc admin policy attach local "$POLICY" --user "$API_KEY" >/dev/null ;;
esac
echo "s3-init: api user exists with policy ${POLICY} only"
access="$(mc anonymous get "local/${BUCKET}")"
case "$access" in
*private*) echo "s3-init: bucket is private (anonymous access: none)" ;;
*) echo "s3-init: bucket is NOT private: $access"; exit 1 ;;
esac
+162
View File
@@ -0,0 +1,162 @@
"""Очистка незавершённых загрузок и сверка PostgreSQL <-> S3.
Запускается отдельным сервисом `janitor` (раз в JANITOR_INTERVAL_SECONDS) или вручную:
python janitor.py --once [--dry-run] [--grace-seconds N]
Проход состоит из двух шагов. Оба трогают только то, что старше grace period
(ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч, минимум 2 x UPLOAD_TIMEOUT_SECONDS): загрузка,
которая ещё идёт, столько не живёт, поэтому свежие и готовые вложения не удаляются.
1. Брошенные намерения: строки state=uploading старше grace. Для каждой (под FOR UPDATE SKIP LOCKED)
удалить объект в S3 (идемпотентно) и затем строку. Если S3 недоступен, строка остаётся
до следующего прохода — указатель на объект не теряется.
2. Сирота в S3: объект под originals/ старше grace, для которого в БД нет строки ни в каком
состоянии. API кладёт объект только после COMMIT строки-намерения, поэтому у живой загрузки
строка всегда есть. Удаляется.
Заодно отчёт: ready-строки, чьего объекта нет в bucket (missing). Их не трогаем — это потеря
данных, её разбирает человек (восстановление из резервной копии), а не автоматическая очистка.
Одновременно работает только один janitor: pg_try_advisory_lock.
"""
import argparse
import json
import logging
import signal
import threading
import time
from datetime import datetime, timedelta, timezone
import psycopg
from psycopg.rows import dict_row
import settings
import storage
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
log = logging.getLogger("messenger.janitor")
JANITOR_LOCK = 20260404
BATCH = 100
STOP = threading.Event()
def connect():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
"""Шаг 1. Возвращает (найдено, удалено)."""
found = deleted = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, object_key FROM attachments
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
ORDER BY created_at LIMIT %s
FOR UPDATE SKIP LOCKED""",
(grace, BATCH),
).fetchall()
found += len(rows)
if dry_run or not rows:
return found, deleted
for row in rows:
storage.delete(row["object_key"]) # недоступен S3 -> исключение, транзакция откатится
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
deleted += 1
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
if len(rows) < BATCH:
return found, deleted
return found, deleted
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
"""Шаг 2. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
started = datetime.now(timezone.utc)
cutoff = started - timedelta(seconds=grace)
scanned = orphans = orphans_deleted = 0
listed = set()
page: list[tuple[str, datetime]] = []
def check(page_items):
nonlocal orphans, orphans_deleted
keys = [key for key, _ in page_items]
known = {r["object_key"] for r in conn.execute(
"SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,)).fetchall()}
for key, modified in page_items:
if key in known or modified >= cutoff:
continue
orphans += 1
if not dry_run:
# Повторная проверка прямо перед удалением: строка могла появиться между запросами.
if conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (key,)).fetchone() is None:
storage.delete(key)
orphans_deleted += 1
log.info("deleted orphan object %s (no row in PostgreSQL)", key)
for key, modified, _size in storage.list_objects():
scanned += 1
listed.add(key)
page.append((key, modified))
if len(page) >= 1000:
check(page)
page = []
if page:
check(page)
missing = [str(r["id"]) for r in conn.execute(
"SELECT id, object_key FROM attachments WHERE state = 'ready' AND updated_at < %s", (started,)).fetchall()
if r["object_key"] not in listed]
if missing:
log.error("ready attachments without object in S3 (not deleted, needs restore): %s", ", ".join(missing[:20]))
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
"missing_objects": len(missing)}
def run_once(grace: int, dry_run: bool) -> dict | None:
started = time.monotonic()
with connect() as conn:
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
log.info("another janitor is running, skip")
return None
try:
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
"stale_uploads_deleted": deleted, **sweep_objects(conn, grace, dry_run)}
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
report["seconds"] = round(time.monotonic() - started, 3)
return report
def main():
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads and orphan S3 objects")
parser.add_argument("--once", action="store_true", help="one pass and exit")
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
args = parser.parse_args()
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
while True:
try:
report = run_once(args.grace_seconds, args.dry_run)
if report is not None:
print(json.dumps(report), flush=True)
failed = False
except (psycopg.Error, storage.StorageUnavailable) as exc:
log.warning("pass failed, will retry: %s", type(exc).__name__)
failed = True
if args.once:
raise SystemExit(1 if failed else 0)
if STOP.wait(settings.JANITOR_INTERVAL_SECONDS):
return
if __name__ == "__main__":
main()
+583
View File
@@ -0,0 +1,583 @@
import base64
import binascii
import hashlib
import json
import logging
import re
import threading
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime
from pathlib import Path
from typing import Literal
from urllib.parse import quote
from uuid import UUID
import uvicorn
from fastapi import FastAPI, Header, Query, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
import attachments
import auth
import db
import settings
import storage
import uploads
from errors import ApiError, bad_request, unauthorized, unavailable
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
log = logging.getLogger("messenger")
DRAINING = threading.Event()
class Server(uvicorn.Server):
"""Остановка по SIGTERM в два шага.
1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS
HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы.
2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы
(не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM,
получал бы 502 от балансировщика.
Повторный сигнал останавливает сразу.
"""
def handle_exit(self, sig, frame):
if DRAINING.is_set():
return super().handle_exit(sig, frame)
DRAINING.set()
log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s",
sig, settings.SHUTDOWN_DRAIN_SECONDS)
threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start()
@asynccontextmanager
async def lifespan(app: FastAPI):
settings.check()
# Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему.
await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS)
# Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3.
Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True)
db.pool.open()
log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s",
settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, settings.SESSION_TTL_SECONDS,
settings.S3_BUCKET)
yield
db.pool.close()
auth.close()
log.info("Stopped: connections closed, data stays in PostgreSQL and Redis")
app = FastAPI(lifespan=lifespan)
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
class CreateUser(BaseModel):
model_config = ConfigDict(extra="forbid")
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH)
@field_validator("display_name", "password")
@classmethod
def validate_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("value cannot be blank")
return value
class User(BaseModel):
id: UUID
username: str
display_name: str
created_at: datetime
class Session(BaseModel):
token: str
token_type: str
expires_at: datetime
user: User
class CreateChat(BaseModel):
model_config = ConfigDict(extra="forbid")
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def validate_title(cls, value: str) -> str:
if not value.strip():
raise ValueError("title cannot be empty")
return value
class Chat(BaseModel):
id: UUID
title: str
member_ids: list[UUID]
created_at: datetime
class ChatPage(BaseModel):
items: list[Chat]
next_cursor: str | None
class CreateMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
text: str = Field(max_length=4000)
# С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением.
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def validate_content(self):
if len(set(self.attachment_ids)) != len(self.attachment_ids):
raise ValueError("attachment_ids must be unique")
if not self.attachment_ids and not self.text.strip():
raise ValueError("text cannot be empty without attachments")
return self
class Message(BaseModel):
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: list[UUID]
class Variant(BaseModel):
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class Attachment(BaseModel):
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: Literal["ready"]
variants: list[Variant]
error_code: str | None
created_at: datetime
updated_at: datetime
class MessagePage(BaseModel):
items: list[Message]
next_cursor: str | None
def error_response(
request: Request,
code: str,
message: str,
status_code: int,
headers: dict | None = None,
):
request_id = request.state.request_id
return JSONResponse(
status_code=status_code,
content={
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
},
headers={"X-Request-Id": request_id, **(headers or {})},
)
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(
request: Request,
exc: RequestValidationError,
):
return error_response(
request,
"invalid_request",
"Invalid request data",
400,
)
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
if exc.status_code == 404:
return error_response(request, "not_found", "Not found", 404)
return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers)
@app.exception_handler(Exception)
async def general_exception_handler(
request: Request,
exc: Exception,
):
return error_response(
request,
"internal_error",
"Internal server error",
500,
)
PUBLIC_ENDPOINTS = {
("POST", "/api/v1/users"),
("POST", "/api/v1/auth/sessions"),
}
@app.middleware("http")
async def request_context(request: Request, call_next):
request_id = str(uuid.uuid4())
request.state.request_id = request_id
started = time.perf_counter()
path = request.url.path
try:
if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS:
request.state.session = await run_in_threadpool(
auth.check_bearer, request.headers.get("Authorization")
)
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
except Exception:
log.exception("Unhandled error, request_id=%s", request_id)
response = error_response(request, "internal_error", "Internal server error", 500)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила
# Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки.
level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO
log.log(
level,
"%s %s -> %s (%.1f ms) request_id=%s",
request.method, path, response.status_code,
(time.perf_counter() - started) * 1000, request_id,
)
return response
def current_user_id(request: Request) -> UUID:
session = getattr(request.state, "session", None)
if session is None:
raise unauthorized("Bearer token is required")
return session.user_id
def encode_cursor(
created_at: datetime,
object_id: UUID,
*,
chat_id: UUID | None = None,
user_id: UUID | None = None,
) -> str:
data = {
"created_at": created_at.isoformat(),
"id": str(object_id),
}
if chat_id is not None:
data["chat_id"] = str(chat_id)
if user_id is not None:
data["user_id"] = str(user_id)
raw = json.dumps(data).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii")
def decode_cursor(cursor: str):
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
data = json.loads(raw.decode("utf-8"))
created_at = datetime.fromisoformat(data["created_at"])
if created_at.tzinfo is None:
raise ValueError("Invalid cursor")
if not isinstance(data["id"], str):
raise ValueError("Invalid cursor")
object_id = UUID(data["id"])
chat_id = UUID(data["chat_id"]) if "chat_id" in data else None
user_id = UUID(data["user_id"]) if "user_id" in data else None
return created_at, object_id, chat_id, user_id
except (
ValueError,
KeyError,
TypeError,
AttributeError,
json.JSONDecodeError,
binascii.Error,
):
raise ValueError("Invalid cursor")
def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None):
if cursor is None:
return None
try:
created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor)
except ValueError:
raise bad_request("Invalid cursor") from None
if cursor_chat_id != chat_id or cursor_user_id != user_id:
raise bad_request("Invalid cursor")
return created_at, object_id
@app.get("/health/live")
def health_live():
return {"status": "ok"}
@app.get("/health/ready")
def health_ready():
if DRAINING.is_set():
raise unavailable("Instance is shutting down")
db.ping()
auth.ping_redis()
return {"status": "ok"}
@app.post("/api/v1/users", response_model=User, status_code=201)
def create_user(user_data: CreateUser):
password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется
return db.create_user(user_data.username, user_data.display_name, password_hash)
@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201)
def create_session(
response: Response,
authorization: str | None = Header(default=None),
):
credentials = auth.parse_basic(authorization)
if credentials is None:
raise auth.login_failed()
username, password = credentials
found = None
if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH:
found = db.get_user_with_password(username)
if not auth.password_matches(found["password_hash"] if found else None, password):
raise auth.login_failed()
token, expires_at = auth.create_session(found["id"])
response.headers["Cache-Control"] = "no-store"
user = {key: found[key] for key in ("id", "username", "display_name", "created_at")}
return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def delete_session(request: Request):
current_user_id(request)
auth.delete_session(request.state.session)
return Response(status_code=204)
@app.get("/api/v1/users/me", response_model=User)
def get_current_user(request: Request):
user = db.get_user(current_user_id(request))
if user is None:
raise unauthorized("Session is invalid or expired")
return user
@app.post("/api/v1/chats", response_model=Chat, status_code=201)
def create_chat(request: Request, chat_data: CreateChat):
user_id = current_user_id(request)
if user_id not in chat_data.member_ids:
raise bad_request("Creator must be a chat member")
if len(chat_data.member_ids) != len(set(chat_data.member_ids)):
raise bad_request("Duplicate member IDs")
return db.create_chat(chat_data.title, chat_data.member_ids)
@app.get("/api/v1/chats", response_model=ChatPage)
def list_chats(
request: Request,
limit: int = Query(default=50, ge=1, le=100),
cursor: str | None = Query(default=None),
):
user_id = current_user_id(request)
after = page_after(cursor, user_id=user_id)
rows = db.list_chats(user_id, after, limit)
items = rows[:limit]
next_cursor = None
if len(rows) > limit:
last = items[-1]
next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id)
return ChatPage(items=items, next_cursor=next_cursor)
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
def get_chat(request: Request, chat_id: UUID):
return db.get_chat(chat_id, current_user_id(request))
def request_fingerprint(data: BaseModel) -> bytes:
"""SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют.
Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []}
равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними.
"""
body = data.model_dump(mode="json")
if not body.get("attachment_ids"):
body.pop("attachment_ids", None)
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).digest()
@app.post(
"/api/v1/chats/{chat_id}/messages",
response_model=Message,
status_code=201,
)
def create_message(
request: Request,
message_data: CreateMessage,
chat_id: UUID,
idempotency_key: str | None = Header(default=None),
):
# С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику)
# вернёт то же сообщение, а не создаст второе. См. db.add_message.
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -")
return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids,
idempotency_key, request_fingerprint(message_data))
@app.get(
"/api/v1/chats/{chat_id}/messages",
response_model=MessagePage,
)
def list_messages(
request: Request,
chat_id: UUID,
limit: int = Query(default=50, ge=1, le=100),
cursor: str | None = Query(default=None),
):
user_id = current_user_id(request)
after = page_after(cursor, chat_id=chat_id)
rows = db.list_messages(chat_id, user_id, after, limit)
items = rows[:limit]
next_cursor = None
if len(rows) > limit:
last = items[-1]
next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id)
return MessagePage(items=items, next_cursor=next_cursor)
@app.post(
"/api/v1/chats/{chat_id}/attachments",
response_model=Attachment,
status_code=201,
)
async def upload_attachment(
request: Request,
chat_id: UUID,
filename: str | None = Query(default=None),
):
"""Синхронная загрузка PNG/JPEG сырыми байтами. 201 — только когда байты в S3 и строка ready в БД.
Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415)
-> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413),
проверка содержимого (415/413) и запись (attachments.store).
"""
user_id = current_user_id(request)
try:
await run_in_threadpool(db.check_member, chat_id, user_id)
name = uploads.validate_filename(filename)
declared = uploads.declared_type(request.headers.get("content-type"))
uploads.declared_length(request.headers.get("content-length"))
except ApiError:
await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке
raise
upload = await uploads.receive(request)
try:
image = await run_in_threadpool(uploads.inspect, upload, declared)
row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image)
finally:
upload.file.close() # временный файл удаляется сразу; оригинал остаётся только в S3
return attachments.public(row)
@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
def get_attachment(request: Request, attachment_id: UUID):
return attachments.public(db.get_attachment(attachment_id, current_user_id(request)))
@app.get("/api/v1/attachments/{attachment_id}/content")
def download_attachment(
request: Request,
attachment_id: UUID,
variant: Literal["original"] = Query(default="original"),
):
"""Байты оригинала через API (без redirect): сначала проверка участника чата, потом S3."""
row = db.get_attachment(attachment_id, current_user_id(request))
try:
stream = storage.open_read(row["object_key"])
except storage.ObjectMissing:
log.error("Object of ready attachment %s is missing in S3", row["id"])
raise unavailable("File is temporarily unavailable") from None
except storage.StorageUnavailable:
raise unavailable("File storage is temporarily unavailable") from None
headers = {
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "inline; filename*=UTF-8''" + quote(row["filename"], safe=""),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
"ETag": '"' + row["sha256"] + '"',
}
return StreamingResponse(storage.iter_chunks(stream), media_type=row["content_type"], headers=headers)
if __name__ == "__main__":
Server(uvicorn.Config(
app,
host=settings.APP_HOST,
port=settings.APP_PORT,
timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS,
access_log=False,
)).run()
+8
View File
@@ -0,0 +1,8 @@
import logging
import db
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
if __name__ == "__main__":
db.wait_and_migrate()
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_by_user ON chat_members (user_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT now(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
);
CREATE INDEX messages_history ON messages (chat_id, created_at, id);
+11
View File
@@ -0,0 +1,11 @@
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8');
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8');
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8');
ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400);
ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400);
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000);
+11
View File
@@ -0,0 +1,11 @@
ALTER TABLE messages ADD COLUMN idempotency_key text
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$');
ALTER TABLE messages ADD COLUMN request_hash bytea
CHECK (octet_length(request_hash) = 32);
ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together
CHECK ((idempotency_key IS NULL) = (request_hash IS NULL));
ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key
UNIQUE (chat_id, sender_id, idempotency_key);
+48
View File
@@ -0,0 +1,48 @@
-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением.
-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать.
-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата.
ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
owner_id uuid NOT NULL,
filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
width integer NOT NULL CHECK (width >= 1),
height integer NOT NULL CHECK (height >= 1),
-- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит.
object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'),
-- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна;
-- ready: объект в S3 сохранён, можно читать и прикреплять.
state text NOT NULL CHECK (state IN ('uploading', 'ready')),
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
CHECK (width::bigint * height <= 40000000),
UNIQUE (id, chat_id),
-- владелец на момент загрузки был участником именно этого чата
FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id)
);
-- janitor ищет старые незавершённые загрузки
CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading';
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, position),
UNIQUE (message_id, attachment_id),
-- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API
FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id),
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id)
);
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
-- С lab4 текст может быть пустым, если есть вложение (проверяет API).
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000);
+29
View File
@@ -0,0 +1,29 @@
fastapi==0.141.1
uvicorn==0.54.0
pydantic==2.13.5
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg-pool==3.3.3
argon2-cffi==25.1.0
redis==6.4.0
annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
argon2-cffi-bindings==26.1.0
cffi==2.1.1
click==8.5.0
h11==0.16.0
idna==3.20
pycparser==3.1
pydantic_core==2.46.5
starlette==1.7.0
typing-inspection==0.4.4
typing_extensions==4.16.0
boto3==1.43.116
botocore==1.43.116
s3transfer==0.19.3
jmespath==1.1.0
python-dateutil==2.9.0.post0
six==1.17.0
urllib3==2.8.0
pillow==12.3.0
+103
View File
@@ -0,0 +1,103 @@
"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init.
Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает.
Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py):
admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init;
messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor.
Значения ключей не печатаются.
"""
import logging
import sys
import time
import urllib.error
import urllib.request
import uuid
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
import settings
logging.basicConfig(level=logging.WARNING)
def client(key_id: str, secret: str):
return boto3.client(
"s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION,
aws_access_key_id=key_id, aws_secret_access_key=secret,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2,
read_timeout=10, retries={"total_max_attempts": 1},
request_checksum_calculation="when_required", response_checksum_validation="when_required"),
)
def code(exc: ClientError) -> str:
return exc.response.get("Error", {}).get("Code", "?")
def anonymous_status(url: str, method: str = "GET") -> int:
try:
with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply:
return reply.status
except urllib.error.HTTPError as exc:
return exc.code
def main() -> int:
bucket = settings.S3_BUCKET
admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"),
settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY"))
api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY)
deadline = time.monotonic() + 90
while True:
try:
admin.list_buckets()
break
except (BotoCoreError, ClientError) as exc:
if time.monotonic() > deadline:
print(f"s3-init: S3 is not ready: {type(exc).__name__}")
return 1
time.sleep(1)
try:
admin.create_bucket(Bucket=bucket)
print(f"s3-init: bucket {bucket} created")
except ClientError as exc:
if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
print(f"s3-init: cannot create bucket: {code(exc)}")
return 1
print(f"s3-init: bucket {bucket} exists")
# Ключ API работает со своим bucket ...
probe = "originals/s3-init-probe-" + uuid.uuid4().hex
api.put_object(Bucket=bucket, Key=probe, Body=b"probe")
api.get_object(Bucket=bucket, Key=probe)["Body"].read()
api.list_objects_v2(Bucket=bucket, MaxKeys=1)
api.delete_object(Bucket=bucket, Key=probe)
print("s3-init: api key can put/get/list/delete objects of its bucket")
# ... и не может администрировать хранилище.
intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8]
try:
api.create_bucket(Bucket=intruder)
admin.delete_bucket(Bucket=intruder)
print("s3-init: api key must NOT be able to create buckets")
return 1
except ClientError as exc:
print(f"s3-init: api key cannot create buckets ({code(exc)})")
# Приватность: анонимные запросы без подписи.
base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket
statuses = {"list": anonymous_status(base + "?list-type=2"),
"get": anonymous_status(base + "/originals/any"),
"put": anonymous_status(base + "/originals/any", "PUT")}
if any(status not in (401, 403) for status in statuses.values()):
print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}")
return 1
print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})")
return 0
if __name__ == "__main__":
sys.exit(main())
+77
View File
@@ -0,0 +1,77 @@
import json
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
def write(name: str, value: str) -> None:
with open(SECRETS / name, "w", encoding="utf-8", newline="") as file:
file.write(value)
(SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore
def bucket_name() -> str:
"""S3_BUCKET из .env (если задан), иначе значение по умолчанию."""
env = ROOT / ".env"
if env.exists():
for line in env.read_text(encoding="utf-8").splitlines():
if line.strip().startswith("S3_BUCKET="):
return line.split("=", 1)[1].strip()
return "messenger-attachments"
def s3_config(bucket: str) -> str:
"""Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket.
Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects)
с суффиксом :<bucket> действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет.
Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются.
"""
def read(name):
return (SECRETS / name).read_text(encoding="utf-8").strip()
config = {"identities": [
{"name": "admin",
"credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
{"name": "messenger-api",
"credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}],
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
]}
return json.dumps(config, indent=2)
def main() -> None:
"""Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes)."""
SECRETS.mkdir(exist_ok=True)
created = []
if not (SECRETS / "postgres_password").exists():
db_password = secrets.token_hex(24)
redis_password = secrets.token_hex(24)
write("postgres_password", db_password)
write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger")
write("cache_url", f"redis://:{redis_password}@redis:6379/0")
write("redis_auth", f"requirepass {redis_password}\n")
created += ["postgres_password", "database_url", "cache_url", "redis_auth"]
if not (SECRETS / "s3_root_password").exists():
# admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init
write("s3_root_user", "root-" + secrets.token_hex(6))
write("s3_root_password", secrets.token_hex(24))
created += ["s3_root_user", "s3_root_password"]
if not (SECRETS / "s3_secret_access_key").exists():
# отдельный ключ API с политикой только на свой bucket (создаёт s3-init)
write("s3_access_key_id", "api-" + secrets.token_hex(8))
write("s3_secret_access_key", secrets.token_hex(24))
created += ["s3_access_key_id", "s3_secret_access_key"]
if not (SECRETS / "s3_config.json").exists():
write("s3_config.json", s3_config(bucket_name()))
created.append("s3_config.json")
if created:
print("Created in secrets/: " + ", ".join(created) + " (values are not printed).")
else:
print("secrets/ is complete, nothing changed.")
if __name__ == "__main__":
main()
+13
View File
@@ -0,0 +1,13 @@
@echo off
rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh).
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then api and janitor, then the balancer. Volumes are kept.
rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
cd /d "%~dp0.."
docker compose restart postgres redis s3 || exit /b 1
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 || exit /b 1
docker compose restart api janitor || exit /b 1
docker compose restart lb || exit /b 1
docker compose up -d --wait --no-recreate --no-build --no-deps api lb || exit /b 1
exit /b 0
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
# Restart hook for tests/resilience.py --mode persistence (Linux/macOS).
# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3), then all api replicas and janitor, then the balancer.
# Volumes are kept (no down -v), nothing is re-seeded.
# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
set -eu
cd "$(dirname "$0")/.."
docker compose restart postgres redis s3
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3
docker compose restart api janitor
docker compose restart lb
docker compose up -d --wait --no-recreate --no-build --no-deps api lb
+6
View File
@@ -0,0 +1,6 @@
@echo off
rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh).
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py.
python "%~dp0stop_one.py"
exit /b %ERRORLEVEL%
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID.
# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd.
set -eu
exec python3 "$(dirname "$0")/stop_one.py"
+29
View File
@@ -0,0 +1,29 @@
import os
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
STOP_TIMEOUT_SECONDS = 8
def main() -> int:
target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower()
if not re.fullmatch(r"[0-9a-f]{12,64}", target):
print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr)
return 2
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
capture_output=True, text=True, check=True).stdout.split()
matches = [container for container in running if container.startswith(target)]
if len(matches) != 1:
print(f"no running api container with id {target}", file=sys.stderr)
return 1
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
print(f"stopped api container {matches[0][:12]}")
return 0
if __name__ == "__main__":
sys.exit(main())
+86
View File
@@ -0,0 +1,86 @@
import os
import socket
from pathlib import Path
def env_str(name: str, default: str) -> str:
return os.environ.get(name, default).strip()
def env_int(name: str, default: int) -> int:
value = int(env_str(name, str(default)))
if value < 1:
raise ValueError(f"{name} must be >= 1")
return value
def read_secret(file_variable: str, plain_variable: str) -> str:
path = env_str(file_variable, "")
if path:
return Path(path).read_text(encoding="utf-8").strip()
return env_str(plain_variable, "")
APP_HOST = env_str("APP_HOST", "0.0.0.0")
APP_PORT = env_int("APP_PORT", 8080)
APP_ENV = env_str("APP_ENV", "lab")
APP_VERSION = env_str("APP_VERSION", "dev")
INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname()
SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600)
SESSION_STORE = env_str("SESSION_STORE", "redis")
DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
DB_POOL_MIN = env_int("DB_POOL_MIN", 1)
DB_POOL_MAX = env_int("DB_POOL_MAX", 10)
DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3)
DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3)
DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000)
REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL")
REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20)
REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500)
MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations")))
SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60)
# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket.
S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333")
S3_REGION = env_str("S3_REGION", "us-east-1")
S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY")
S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1)
S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5)
S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2)
S3_POOL_MAX = env_int("S3_POOL_MAX", 20)
# Вложения: лимиты из контракта и защита памяти/времени.
MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)
MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000)
UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60)
UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10)
DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2)
UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads")
# Очистка незавершённых загрузок (janitor.py).
ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600)
JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300)
SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3)
SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10)
def check() -> None:
"""Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе."""
if not DATABASE_URL:
raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE")
if not REDIS_URL:
raise RuntimeError("Redis address is not set: CACHE_URL_FILE")
if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY:
raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE")
if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS:
raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS")
if SESSION_STORE != "redis":
raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution")
+153
View File
@@ -0,0 +1,153 @@
"""Приватное S3-хранилище (SeaweedFS) для байтов вложений.
API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket
(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер,
имя файла пользователя в ключ не попадает.
"""
import base64
import logging
from datetime import datetime
from typing import BinaryIO, Iterator
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
import settings
log = logging.getLogger("messenger.storage")
ORIGINALS_PREFIX = "originals/"
READ_CHUNK = 64 * 1024
class StorageUnavailable(Exception):
"""S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей."""
class ObjectMissing(Exception):
"""Объекта нет в bucket (NoSuchKey / 404)."""
def _make_client(connect_timeout: float, read_timeout: float, attempts: int):
return boto3.client(
"s3",
endpoint_url=settings.S3_ENDPOINT,
region_name=settings.S3_REGION,
aws_access_key_id=settings.S3_ACCESS_KEY_ID or None,
aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"}, # http://s3:8333/<bucket>/<key>
connect_timeout=connect_timeout,
read_timeout=read_timeout,
retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую
max_pool_connections=settings.S3_POOL_MAX,
# Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно.
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
),
)
_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS)
# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor.
_quick_client = _make_client(1, 2, 1)
def _is_missing(exc: ClientError) -> bool:
code = exc.response.get("Error", {}).get("Code", "")
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
return code in ("NoSuchKey", "404", "NotFound") or status == 404
def _unavailable(action: str, exc: Exception) -> StorageUnavailable:
detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else ""
log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail)
return StorageUnavailable()
def new_key(object_id: str) -> str:
return ORIGINALS_PREFIX + object_id
def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
"""Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект."""
body.seek(0)
try:
_client.put_object(
Bucket=settings.S3_BUCKET,
Key=key,
Body=body,
ContentLength=size,
ContentType=content_type,
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
)
except (BotoCoreError, ClientError) as exc:
raise _unavailable("put", exc) from None
def open_read(key: str):
"""GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после."""
try:
return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"]
except ClientError as exc:
if _is_missing(exc):
raise ObjectMissing(key) from None
raise _unavailable("get", exc) from None
except BotoCoreError as exc:
raise _unavailable("get", exc) from None
def iter_chunks(stream) -> Iterator[bytes]:
"""Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание."""
try:
while True:
chunk = stream.read(READ_CHUNK)
if not chunk:
return
yield chunk
except (BotoCoreError, OSError) as exc:
# Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length).
log.warning("S3 stream interrupted: %s", type(exc).__name__)
raise
finally:
stream.close()
def delete(key: str, quick: bool = False) -> None:
"""Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа."""
try:
(_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key)
except (BotoCoreError, ClientError) as exc:
raise _unavailable("delete", exc) from None
def exists(key: str) -> bool:
try:
_client.head_object(Bucket=settings.S3_BUCKET, Key=key)
return True
except ClientError as exc:
if _is_missing(exc):
return False
raise _unavailable("head", exc) from None
except BotoCoreError as exc:
raise _unavailable("head", exc) from None
def list_objects(prefix: str = ORIGINALS_PREFIX) -> Iterator[tuple[str, datetime, int]]:
"""Все объекты под prefix: (key, last_modified, size). Постранично, по 1000."""
paginator = _client.get_paginator("list_objects_v2")
try:
for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix):
for item in page.get("Contents", []):
yield item["Key"], item["LastModified"], item["Size"]
except (BotoCoreError, ClientError) as exc:
raise _unavailable("list", exc) from None
def ping() -> None:
try:
_client.head_bucket(Bucket=settings.S3_BUCKET)
except (BotoCoreError, ClientError) as exc:
raise _unavailable("head_bucket", exc) from None
+78
View File
@@ -0,0 +1,78 @@
import json
import platform
import sys
from common import call, check, compose_output, docker, new_user, seen_instances
def main():
info = json.loads(docker("info", "--format", "{{json .}}"))
print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
f"client Python {platform.python_version()}")
print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, "
f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, "
f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}")
print()
print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip())
for one_shot in ("migrate", "s3-init"):
status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip()
check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0")
config = json.loads(compose_output("config", "--format", "json"))
services = config["services"]
containers = compose_output("ps", "-q").split()
rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()]
print()
print("container image cpus memory host ports networks")
api_count = 0
for row in sorted(rows, key=lambda r: r["Name"]):
labels = row["Config"]["Labels"]
service = labels.get("com.docker.compose.service")
limits = row["HostConfig"]
cpus = limits.get("NanoCpus", 0) / 1e9
memory = limits.get("Memory", 0) // 2**20
ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values()
for b in (bindings or [])})
networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"])
print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB "
f"{','.join(ports) or '-':<23} {','.join(networks)}")
check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit")
if service == "api":
api_count += 1
check(not ports, "api must not publish ports: everything goes through the balancer")
if service == "lb":
check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only")
if service in ("postgres", "redis", "s3", "janitor"):
check(not ports and networks == ["data"], f"{service} must be only in the internal data network")
check(api_count >= 2, "need at least 2 api replicas")
check(all("container_name" not in s for s in services.values()), "container_name prevents scaling")
check(config["networks"]["data"].get("internal") is True, "data network must be internal")
check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis")
print()
for name, service in sorted(services.items()):
secrets = sorted(s["source"] for s in service.get("secrets", []))
print(f"secrets of {name}: {secrets or '-'}")
check(not services["lb"].get("secrets"), "lb gets no secrets")
check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url")
for name in ("api", "janitor"):
names = {s["source"] for s in services[name]["secrets"]}
check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys")
check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key")
user = new_user("stack")
served = seen_instances(user["token"])
print()
print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}")
check(len(served) >= 2, "requests are not balanced")
health = call("GET", "/health/ready", 200)
print(f"/health/ready -> {health.status} from {health.instance}")
print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every "
f"container, postgres/redis/s3/janitor only in the internal data network, "
f"S3 admin keys only in s3 and s3-init")
if __name__ == "__main__":
sys.exit(main())
+293
View File
@@ -0,0 +1,293 @@
import json
import os
import platform
import re
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
PY = sys.executable
STEPS = [
("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]),
("stack", "stand layout: replicas, ports, limits, networks, secrets, S3", [PY, "student-tests/check_stack.py"]),
("smoke", "public smoke via the balancer (make test), with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]),
("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids",
[PY, "student-tests/test_attachments.py"]),
("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap",
[PY, "student-tests/test_s3_security.py"]),
("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage",
[PY, "student-tests/test_consistency.py"]),
("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas",
[PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]),
("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]),
("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]),
("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]),
("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]),
("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]),
("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]),
("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]),
("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)",
[PY, "student-tests/run_failover.py"]),
("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL,
"--duration", "30", "--concurrency", "4",
"--output", "evidence/load.json"]),
("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]),
("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)",
[PY, "student-tests/run_persistence.py"]),
]
def git(*args):
try:
return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
except (OSError, subprocess.CalledProcessError):
return ""
def commit():
head = git("rev-parse", "--short", "HEAD") or "unknown"
changed = [line for line in git("status", "--porcelain").splitlines()
if not line[3:].startswith(("evidence/", "REPORT.md"))]
return head + (" + uncommitted changes" if changed else "")
def run_step(name, description, command):
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL)
print(f"=== {name}: {description}", flush=True)
started = time.monotonic()
result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
seconds = time.monotonic() - started
output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n")
shown = " ".join(Path(part).name if part == PY else part for part in command)
header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n"
f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n")
(EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8")
tail = [line for line in output.splitlines() if line.strip()][-3:]
for line in tail:
print(" " + line)
print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True)
return result.returncode == 0, seconds
def load_json(name):
path = EVIDENCE / name
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None
def measurements():
"""Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md)."""
lines = []
try:
info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True,
text=True, check=True).stdout)
docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM"
except (OSError, ValueError, subprocess.CalledProcessError):
docker_vm = "unknown"
lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона "
f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB "
f"каждая, postgres и s3 (SeaweedFS) по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, "
f"janitor 0,25 CPU / 192 MiB.")
lines.append("")
failover = EVIDENCE / "failover.txt"
results = []
if failover.exists():
text = failover.read_text(encoding="utf-8")
stopped = re.findall(r"the hook stopped (\w+)", text)
results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)]
if results:
lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):")
lines.append("")
lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток "
"(повторены с тем же ключом) | Первая запись после hook, с |")
lines.append("| --- | --- | --- | --- | --- | --- |")
for number, (result, container) in enumerate(zip(results, stopped), start=1):
lines.append(f"| {number} | {container} | {result['observed_instances_before']} | "
f"{result['successful_writes']} | {result['failed_attempts']} | "
f"{result['first_recovery_seconds']} |")
lines.append("")
memory = load_json("upload-memory.json")
if memory:
lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики "
f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). "
f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:")
lines.append("")
lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | "
"Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |")
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |")
for run in memory["runs"]:
for n, r in enumerate(run["replicas"]):
head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | "
f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |"
cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-"
lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | "
f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |")
naive = memory["single_file_extra_rss_mib"]
lines.append("")
lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в "
f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка "
f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в "
f"{memory['api_limit_mib']} MiB.")
lines.append("")
metrics = load_json("failover-metrics.json")
if metrics:
lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи "
"без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):")
lines.append("")
lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок "
"(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | "
"Вернулась в балансировку после docker start, с |")
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |")
for r in metrics["runs"]:
how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)"
codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-"
lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | "
f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | "
f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |")
lines.append("")
lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика "
"после остановки не ответила ни разу.")
lines.append("")
rolling = load_json("rolling-update.json")
if rolling:
during = rolling["during"]
codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-"
lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> "
f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за "
f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, "
f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); "
f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено "
f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.")
lines.append("")
load = load_json("load.json")
if load:
latency = load["successful_latency_ms"]
lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, "
f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных "
f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных "
f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.")
lines.append("")
persistence = EVIDENCE / "persistence.txt"
restart_seconds = None
if persistence.exists():
found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8"))
restart_seconds = found.group(1) if found else None
impacts = []
if metrics:
for mode, what, expected in (
("stop", "`docker stop` реплики (SIGTERM) под нагрузкой",
"ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"),
("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой",
"единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")):
runs = [r for r in metrics["runs"] if r["mode"] == mode]
if runs:
impacts.append(
f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); "
f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | "
f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; "
f"потерь 0, дублей 0; реплика вернулась через "
f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |")
if failover.exists() and results:
impacts.append(
f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, "
f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: "
f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через "
f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |")
if rolling:
impacts.append(
f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | "
f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |")
if restart_seconds:
impacts.append(
f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и "
f"прежний токен живы | "
f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |")
consistency = EVIDENCE / "consistency.txt"
if consistency.exists():
text = consistency.read_text(encoding="utf-8")
timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text)
if len(timings) >= 2:
impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, "
f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | "
f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |")
impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за "
f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |")
if "PASS: S3 written but DB not committed" in text:
impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | "
"вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | "
"GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | "
"мусора не осталось; повторный проход ничего не делает |")
if "PASS: an interrupted upload leaves no row" in text:
impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | "
"строк и объектов не прибавилось, временных файлов 0 | мгновенно |")
if impacts:
lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |")
lines.append("| --- | --- | --- | --- |")
lines.extend(impacts)
lines.append("")
return "\n".join(lines).strip()
def update_report(block):
report = ROOT / "REPORT.md"
text = report.read_text(encoding="utf-8")
begin, end = "<!-- measurements:begin -->", "<!-- measurements:end -->"
if begin not in text or end not in text:
print("REPORT.md has no measurement markers, only evidence/summary.md was written")
return
head, rest = text.split(begin, 1)
_, tail = rest.split(end, 1)
report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8")
print("REPORT.md: measurement tables updated")
def main():
args = sys.argv[1:]
names = [name for name, _, _ in STEPS]
if args[:1] == ["--from"]:
if len(args) < 2 or args[1] not in names:
raise SystemExit("usage: --from <step>; steps: " + ", ".join(names))
selected = names[names.index(args[1]):]
else:
unknown = [a for a in args if a not in names]
if unknown:
raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names))
selected = args or names
EVIDENCE.mkdir(exist_ok=True)
results = []
for name, description, command in STEPS:
if name not in selected:
continue
ok, seconds = run_step(name, description, command)
results.append((name, description, ok, seconds))
if not ok:
print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.")
print(f"After fixing: python student-tests/collect_evidence.py --from {name}")
break
block = measurements()
summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |",
"| --- | --- | --- | --- | --- |"]
summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results]
(EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8")
print("evidence/summary.md written")
passed = all(ok for _, _, ok, _ in results)
if passed:
update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков
raise SystemExit(0 if passed else 1)
if __name__ == "__main__":
main()
+324
View File
@@ -0,0 +1,324 @@
import base64
import csv
import hashlib
import io
import json
import os
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv")
PASSWORD = "Test-password-123"
class Response:
def __init__(self, status, headers, data):
self.status = status
self.headers = {name.lower(): value for name, value in headers.items()}
self.data = data
try:
self.json = json.loads(data.decode("utf-8")) if data else None
except ValueError:
self.json = None
@property
def instance(self):
"""X-Instance-Id: какая реплика API ответила (ID её контейнера)."""
return self.headers.get("x-instance-id")
def check(condition, message):
if not condition:
raise AssertionError(message)
def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30):
"""HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError.
Заодно проверяет то, что контракт требует от любого ответа:
X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401.
"""
all_headers = dict(headers or {})
data = raw
if body is not None:
data = json.dumps(body).encode("utf-8")
if data is not None:
all_headers.setdefault("Content-Type", "application/json")
if token is not None:
all_headers["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
try:
with urllib.request.urlopen(request, timeout=timeout) as reply:
response = Response(reply.status, reply.headers, reply.read())
except urllib.error.HTTPError as error:
response = Response(error.code, error.headers, error.read())
expected = expected if isinstance(expected, tuple) else (expected,)
check(response.status in expected,
f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}")
request_id = response.headers.get("x-request-id")
check(request_id, f"{method} {path}: no X-Request-Id header")
check(response.instance, f"{method} {path}: no X-Instance-Id header")
if response.status >= 400:
error = (response.json or {}).get("error") or {}
check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format")
check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header")
if response.status == 401:
check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate")
return response
def try_call(method, path, *, body=None, token=None, headers=None, timeout=2):
"""Запрос без проверок и без исключений — для нагрузки во время отказа.
Возвращает Response или None, если соединение оборвалось / вышел timeout.
"""
all_headers = dict(headers or {})
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
all_headers["Content-Type"] = "application/json"
if token is not None:
all_headers["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
try:
with urllib.request.urlopen(request, timeout=timeout) as reply:
return Response(reply.status, reply.headers, reply.read())
except urllib.error.HTTPError as error:
try:
return Response(error.code, error.headers, error.read())
except OSError:
return None
except (urllib.error.URLError, OSError):
return None
def basic(username, password):
"""Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8."""
return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
def login(username, password, expected=201):
return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)})
def new_user(prefix="user", password=PASSWORD):
"""Регистрирует пользователя и входит. Возвращает dict: id, username, password, token."""
username = f"{prefix}_{uuid.uuid4().hex[:10]}"
user = call("POST", "/api/v1/users", 201,
body={"username": username, "display_name": prefix.title(), "password": password}).json
session = login(username, password).json
return {**user, "password": password, "token": session["token"]}
def new_chat(owner, *others, title="Test chat"):
member_ids = [owner["id"]] + [user["id"] for user in others]
return call("POST", "/api/v1/chats", 201, token=owner["token"],
body={"title": title, "member_ids": member_ids}).json
def new_key():
return uuid.uuid4().hex
def send(user, chat_id, text, expected=201, key=None):
"""POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный."""
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
body={"text": text}, headers={"Idempotency-Key": key or new_key()})
def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None):
"""POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся."""
body = {"text": text}
if attachment_ids is not None:
body["attachment_ids"] = attachment_ids
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
body=body, headers={"Idempotency-Key": key or new_key()})
def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=201, timeout=60):
"""POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра."""
query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="")
headers = {"Content-Type": content_type} if content_type is not None else {}
return call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data,
token=user["token"] if user else None, headers=headers, timeout=timeout)
def read_history(user, chat_id, limit=100):
"""Вся история чата по страницам. Возвращает (сообщения, число_страниц)."""
items, pages, cursor = [], 0, None
while True:
query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "")
page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json
items += page["items"]
pages += 1
cursor = page["next_cursor"]
if cursor is None:
return items, pages
def wait_ready(seconds=90):
deadline = time.monotonic() + seconds
while True:
try:
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply:
if reply.status == 200:
return
except (urllib.error.URLError, OSError):
pass
check(time.monotonic() < deadline, f"API is not ready after {seconds} s")
time.sleep(0.5)
def compose(*args, env=None):
"""docker compose <args> в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам."""
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def compose_output(*args, env=None):
"""docker compose <args> и его вывод (для logs, ps)."""
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env)
return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace")
def docker(*args, check_exit=True):
"""docker <args>, возвращает stdout."""
result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit)
return result.stdout.decode("utf-8", "replace").strip()
def api_containers(include_stopped=False):
"""Полные ID контейнеров сервиса api (по умолчанию только запущенные)."""
args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"]
return sorted(compose_output(*args).split())
def health(container):
"""Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''."""
return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container,
check_exit=False)
def lb_servers():
"""Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}."""
with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply:
rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace"))))
header = [name.lstrip("# ") for name in rows[0]]
servers = {}
for row in rows[1:]:
item = dict(zip(header, row))
if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"):
servers[item["svname"]] = item.get("status", "")
return servers
def lb_up_count():
"""Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP)."""
return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP"))
def seen_instances(token, requests=20):
"""Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me."""
return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)}
def wait_balanced(replicas, seconds=60):
"""Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке."""
deadline = time.monotonic() + seconds
while True:
running = api_containers()
healthy = [container for container in running if health(container) == "healthy"]
try:
up = lb_up_count()
except OSError:
up = 0
if len(healthy) >= replicas and up >= replicas:
return running
check(time.monotonic() < deadline,
f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}")
time.sleep(0.5)
def redis_cli(*args):
"""redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод."""
command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args)
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command],
cwd=ROOT, capture_output=True, check=True)
return result.stdout.decode("utf-8", "replace").strip()
def session_key(token):
"""Ключ сессии в Redis, как его считает auth.py: session:<SHA-256 токена>."""
return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
def psql(sql):
"""Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки).
Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя.
"""
result = subprocess.run(
["docker", "compose", "exec", "-T", "postgres",
"psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql],
cwd=ROOT, capture_output=True, check=False,
)
output = result.stdout.decode("utf-8", "replace").strip()
error = result.stderr.decode("utf-8", "replace").strip()
return result.returncode, output, error
def app_python(code, *args, service="api", env=None):
"""Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API).
Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON).
"""
command = ["docker", "compose", "exec", "-T"]
for name, value in (env or {}).items():
command += ["-e", f"{name}={value}"]
command += [service, "python", "-", *map(str, args)]
result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False)
output = result.stdout.decode("utf-8", "replace").strip()
check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
return output
def janitor(*args, env=None):
"""Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт."""
command = ["docker", "compose", "exec", "-T"]
for name, value in (env or {}).items():
command += ["-e", f"{name}={value}"]
command += ["janitor", "python", "janitor.py", "--once", *map(str, args)]
result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False)
output = result.stdout.decode("utf-8", "replace").strip()
check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
return json.loads(output.splitlines()[-1])
def sql_value(query):
"""Одно значение из PostgreSQL (psql -A -t)."""
code, output, error = psql(query)
check(code == 0, f"psql failed: {error[-300:]}")
return output
def run_tests(tests):
"""Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал."""
failed = 0
for test in tests:
try:
test()
except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста
failed += 1
print(f"FAIL: {test.__name__}: {exc}")
print(f"{len(tests) - failed} passed, {failed} failed")
raise SystemExit(1 if failed else 0)
+16
View File
@@ -0,0 +1,16 @@
from common import call, compose_output, new_user
def main():
user = new_user("demo")
print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:")
for number in range(1, 9):
reply = call("GET", "/api/v1/users/me", 200, token=user["token"])
print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}")
print()
print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip())
if __name__ == "__main__":
main()
+28
View File
@@ -0,0 +1,28 @@
import subprocess
from common import ROOT
OUTPUT = ROOT / "evidence" / "explain-history.txt"
def main():
sql = (ROOT / "student-tests" / "explain.sql").read_bytes()
result = subprocess.run(
["docker", "compose", "exec", "-T", "postgres",
"psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"],
cwd=ROOT, input=sql, capture_output=True, check=False,
)
OUTPUT.parent.mkdir(exist_ok=True)
OUTPUT.write_bytes(result.stdout + result.stderr)
if result.returncode != 0:
print(result.stderr.decode("utf-8", "replace"))
raise SystemExit("FAIL: psql returned an error, see output above")
for line in result.stdout.decode("utf-8", "replace").splitlines():
if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line:
print(line)
print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}")
if __name__ == "__main__":
main()
+76
View File
@@ -0,0 +1,76 @@
\pset footer off
BEGIN;
CREATE TEMP TABLE seed_chats AS
SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size
FROM generate_series(0, 50) AS n;
CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id;
INSERT INTO users (id, username, display_name, password_hash)
SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash'
FROM seed_user;
INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats;
INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'),
timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second'
FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g;
ANALYZE messages;
SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset
SELECT created_at AS cursor_time, id AS cursor_id FROM messages
WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
\echo '=== Data volume'
SELECT count(*) AS messages_total,
count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat
FROM messages;
\echo ''
\echo '=== History index'
SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history';
\echo ''
\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page'
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT id, chat_id, sender_id, text, created_at FROM messages
WHERE chat_id = :'big_chat'
ORDER BY created_at, id LIMIT 51;
\echo ''
\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)'
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT id, chat_id, sender_id, text, created_at FROM messages
WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
ORDER BY created_at, id LIMIT 51;
\echo ''
\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message'
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT a.attachment_id FROM message_attachments a
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
FROM messages m
WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
ORDER BY m.created_at, m.id LIMIT 51;
\echo ''
\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)'
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT id, chat_id, sender_id, text, created_at FROM messages
WHERE chat_id = :'big_chat'
ORDER BY created_at, id OFFSET 10000 LIMIT 51;
\echo ''
\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)'
DROP INDEX messages_history;
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
SELECT id, chat_id, sender_id, text, created_at FROM messages
WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
ORDER BY created_at, id LIMIT 51;
ROLLBACK;
\echo ''
\echo '=== ROLLBACK: test data removed, index is back'
SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%';
+129
View File
@@ -0,0 +1,129 @@
import json
import platform
import sys
import threading
import time
from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced
from loadgen import Load, summary
WARMUP_SECONDS = 3
AFTER_SECONDS = 10
OUTPUT = ROOT / "evidence" / "failover-metrics.json"
class LbWatch:
"""Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP)."""
def __init__(self):
self.points, self.stopping = [], threading.Event()
self.thread = threading.Thread(target=self._run, daemon=True)
def _run(self):
while not self.stopping.is_set():
try:
self.points.append((time.monotonic(), lb_up_count()))
except OSError:
pass
time.sleep(0.1)
def start(self):
self.thread.start()
def stop(self):
self.stopping.set()
self.thread.join(timeout=3)
def first_drop(self, after, replicas):
return next((t for t, up in self.points if t >= after and up < replicas), None)
def measure(container, mode, replicas):
wait_balanced(replicas)
load, watch = Load(), LbWatch()
watch.start()
load.start()
time.sleep(WARMUP_SECONDS)
t0 = time.monotonic()
if mode == "stop":
docker("stop", "--time", "8", container)
else:
docker("kill", container)
command_seconds = time.monotonic() - t0
time.sleep(max(0.0, AFTER_SECONDS - command_seconds))
end = t0 + AFTER_SECONDS
load.stop()
watch.stop()
errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]]
errors_end = max((e[0] + e[5] for e in errors), default=None)
drop = watch.first_drop(t0, replicas)
served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]]
target_hits = sum(1 for instance in served_after if instance and container.startswith(instance))
integrity = load.verify()
started = time.monotonic()
docker("start", container)
wait_balanced(replicas)
rejoin_seconds = time.monotonic() - started
return {
"replica": container[:12],
"mode": mode,
"command_seconds": round(command_seconds, 2),
"lb_excluded_after_seconds": round(drop - t0, 2) if drop else None,
"errors": len(errors),
"error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"],
"errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0,
"before": summary(load, t0 - WARMUP_SECONDS, t0),
"after": summary(load, t0, end),
"stopped_replica_answers_after_stop": target_hits,
"integrity": integrity,
"rejoin_seconds": round(rejoin_seconds, 2),
}
def main():
modes = sys.argv[1:] or ["stop", "kill"]
check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]")
containers = api_containers()
replicas = len(containers)
check(replicas >= 2, "need at least 2 running api replicas")
print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, "
f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and "
f"{AFTER_SECONDS} s after the failure")
runs = []
for container in containers:
for mode in modes:
print(f"--- {mode} {container[:12]} ...")
run = measure(container, mode, replicas)
runs.append(run)
print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after "
f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, "
f"stopped after {run['errors_stopped_after_seconds']} s")
print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; "
f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; "
f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}")
print(f" confirmed {run['integrity']['confirmed_writes']}, in history "
f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; "
f"back in the balancer {run['rejoin_seconds']} s after docker start")
check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered")
check(run["after"]["write_rps"] > 0, "no successful writes after the failure")
print()
print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s")
for r in runs:
print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} "
f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} "
f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}")
OUTPUT.parent.mkdir(exist_ok=True)
OUTPUT.write_text(json.dumps({
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS,
"after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()},
"runs": runs,
}, indent=2) + "\n", encoding="utf-8")
print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; "
f"saved {OUTPUT.relative_to(ROOT)}")
if __name__ == "__main__":
main()
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

+74
View File
@@ -0,0 +1,74 @@
"""Синтетические изображения для тестов вложений: только стандартная библиотека.
PNG собирается вручную (сигнатура, IHDR, IDAT, IEND), поэтому можно получить любой размер
в пикселях и байтах, а также испортить конкретный байт. JPEG берётся из fixtures/.
"""
import os
import struct
import zlib
from pathlib import Path
FIXTURES = Path(__file__).resolve().parent / "fixtures"
ROOT = Path(__file__).resolve().parents[1]
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
def chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes:
"""PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой).
pad_to — дополнить файл служебным чанком prVt до точного размера в байтах.
idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое).
"""
channels = 1 if gray else 3
header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace)
if idat is None:
compressor = zlib.compressobj(6)
parts = []
# Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход.
passes = [(width, height)] if not interlace else [
((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7]
for pass_width, pass_height in passes:
if pass_width <= 0 or pass_height <= 0:
continue
row = b"\x00" + bytes(pass_width * channels)
for _ in range(pass_height):
parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row))
parts.append(compressor.flush())
idat = b"".join(parts)
body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat)
if pad_to is not None:
need = pad_to - len(body) - 12 - 12 # чанк-заполнитель + IEND
assert need >= 0, "image is already larger than pad_to"
body += chunk(b"prVt", bytes(need))
return body + chunk(b"IEND", b"")
def noise_png(target_bytes: int) -> bytes:
"""Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями)."""
side = int(((target_bytes - 4096) / 3) ** 0.5)
return png(side, side, noise=True)
def sample_png() -> bytes:
return (ROOT / "tests/fixtures/sample.png").read_bytes()
def sample_jpeg() -> bytes:
return (FIXTURES / "sample.jpg").read_bytes()
def progressive_jpeg() -> bytes:
return (FIXTURES / "progressive.jpg").read_bytes()
def flip_byte(data: bytes, position: int) -> bytes:
return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:]
GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"
+102
View File
@@ -0,0 +1,102 @@
import math
import threading
import time
from common import check, new_chat, new_key, new_user, read_history, try_call
class Load:
def __init__(self, writers=4, timeout=2):
self.alice, self.bob = new_user("load_a"), new_user("load_b")
self.chat = new_chat(self.alice, self.bob, title="load")
self.writers, self.timeout = writers, timeout
self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с)
self.confirmed = [] # сообщения, на которые пришёл 201
self.lock = threading.Lock()
self.stopping = threading.Event()
self.threads = []
def _record(self, kind, started, reply, ok):
code = reply.status if reply is not None else "conn"
instance = reply.instance if reply is not None else None
with self.lock:
self.events.append((started, kind, ok, code, instance, time.monotonic() - started))
def _writer(self, number):
path = f"/api/v1/chats/{self.chat['id']}/messages"
sent = 0
key, text = new_key(), f"load-{number}-{sent}"
while not self.stopping.is_set():
started = time.monotonic()
reply = try_call("POST", path, body={"text": text}, token=self.alice["token"],
headers={"Idempotency-Key": key}, timeout=self.timeout)
ok = reply is not None and reply.status == 201
self._record("write", started, reply, ok)
if ok:
with self.lock:
self.confirmed.append(reply.json)
sent += 1
key, text = new_key(), f"load-{number}-{sent}"
else:
time.sleep(0.05) # и повторить тот же text с тем же ключом
def _reader(self):
path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20"
while not self.stopping.is_set():
started = time.monotonic()
reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout)
self._record("read", started, reply, reply is not None and reply.status == 200)
time.sleep(0.05)
def start(self):
self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)]
self.threads.append(threading.Thread(target=self._reader, daemon=True))
for thread in self.threads:
thread.start()
def stop(self):
self.stopping.set()
for thread in self.threads:
thread.join(timeout=self.timeout + 5)
def window(self, start, end, kind=None):
with self.lock:
return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)]
def verify(self):
"""Все подтверждённые сообщения есть в истории, id и тексты не повторяются."""
history, _ = read_history(self.bob, self.chat["id"])
ids = [message["id"] for message in history]
texts = [message["text"] for message in history]
by_id = {message["id"]: message for message in history}
check(len(ids) == len(set(ids)), "duplicated message ids in history")
check(len(texts) == len(set(texts)), "a retried request created a duplicate message")
lost = [m for m in self.confirmed if by_id.get(m["id"]) != m]
check(not lost, f"{len(lost)} confirmed messages are lost or changed")
return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0}
def summary(load, start, end):
"""Успешные RPS, ошибки и задержки на отрезке [start, end)."""
events = load.window(start, end)
writes = [e for e in events if e[1] == "write"]
ok_writes = sorted(e[5] for e in writes if e[2])
errors = [e for e in events if not e[2]]
seconds = max(end - start, 1e-6)
def percentile(p):
return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None
codes = {}
for e in errors:
codes[str(e[3])] = codes.get(str(e[3]), 0) + 1
return {
"seconds": round(seconds, 2),
"write_rps": round(len(ok_writes) / seconds, 1),
"reads": len([e for e in events if e[1] == "read"]),
"errors": len(errors),
"error_codes": codes,
"write_p50_ms": percentile(0.50),
"write_p95_ms": percentile(0.95),
"write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None,
}
+29
View File
@@ -0,0 +1,29 @@
import subprocess
from common import ROOT, login, new_user, redis_cli, session_key
def without_password():
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"],
cwd=ROOT, capture_output=True, check=False)
return (result.stdout + result.stderr).decode("utf-8", "replace").strip()
def main():
user = new_user("info")
token = login(user["username"], user["password"]).json["token"]
key = session_key(token)
print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}")
print(f"TTL of the key, seconds: {redis_cli('ttl', key)}")
keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines()
print(f"sessions in Redis: {len(keys)}")
for name in ("appendonly", "appendfsync", "maxmemory-policy"):
print("config", " = ".join(redis_cli("config", "get", name).split()))
for line in redis_cli("info", "persistence").splitlines():
if line.startswith(("aof_enabled", "aof_last_write_status")):
print(line)
print(f"without password: {without_password()}")
if __name__ == "__main__":
main()
+88
View File
@@ -0,0 +1,88 @@
import json
import os
import time
from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced
from loadgen import Load, summary
OUTPUT = ROOT / "evidence" / "rolling-update.json"
def env_of(container):
lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines()
return dict(line.split("=", 1) for line in lines if "=" in line)
def wait_new_replica(container, up_needed, seconds=90):
deadline = time.monotonic() + seconds
while health(container) != "healthy" or lb_up_count() < up_needed:
check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s")
time.sleep(0.2)
def main():
old = api_containers()
replicas = len(old)
check(replicas >= 2, "need at least 2 running api replicas")
wait_balanced(replicas)
old_version = env_of(old[0]).get("APP_VERSION")
version = time.strftime("rolling-%Y%m%d-%H%M%S")
env = dict(os.environ, APP_VERSION=version)
print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}")
started = time.monotonic()
compose("build", "api", "migrate", env=env)
print(f"image built in {time.monotonic() - started:.1f} s")
migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env)
print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1]
.split("messenger.db: ")[-1])
load = Load()
load.start()
time.sleep(3)
t0 = time.monotonic()
steps = []
for container in old:
step_start = time.monotonic()
before = set(api_containers())
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env)
new = sorted(set(api_containers()) - before)
check(len(new) == 1, f"expected one new container, got {len(new)}")
wait_new_replica(new[0], replicas + 1)
ready = time.monotonic() - step_start
docker("stop", "--time", "15", container)
docker("rm", container)
steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2),
"step_seconds": round(time.monotonic() - step_start, 2)})
print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, "
f"step {steps[-1]['step_seconds']} s")
t1 = time.monotonic()
time.sleep(3)
load.stop()
final = api_containers()
versions = sorted({env_of(c).get("APP_VERSION") for c in final})
integrity = load.verify()
during = summary(load, t0, t1)
before_stats = summary(load, t0 - 3, t0)
print(f"now: replicas {[c[:12] for c in final]}, versions {versions}")
print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; "
f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, "
f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s")
print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, "
f"lost 0, duplicates 0")
check(len(final) == replicas and not set(final) & set(old), "old replicas are still running")
check(versions == [version], "not every replica runs the new version")
OUTPUT.parent.mkdir(exist_ok=True)
OUTPUT.write_text(json.dumps({
"from_version": old_version, "to_version": version, "replicas": replicas,
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2},
"steps": steps, "update_seconds": round(t1 - t0, 2),
"before": before_stats, "during": during, "integrity": integrity,
}, indent=2) + "\n", encoding="utf-8")
print(f"PASS: all {replicas} replicas replaced one by one under load, "
f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}")
if __name__ == "__main__":
main()
+55
View File
@@ -0,0 +1,55 @@
import os
import subprocess
import sys
from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced
HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
def ps():
return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()
def aim(wanted):
"""Для 2 реплик: последним перед тестом должна ответить реплика из wanted."""
probe = call("GET", "/health/live", 200).instance
if not any(container.startswith(probe) for container in wanted):
call("GET", "/health/live", 200) # теперь последней ответила другая реплика
def main():
containers = api_containers()
check(len(containers) >= 2, "need at least 2 running api replicas")
print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}")
stopped_once = set()
for attempt in range(1, 7):
wait_balanced(len(containers))
if len(containers) == 2:
aim(set(containers) - stopped_once)
print(f"=== run {attempt}: api containers before")
print(ps())
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover",
"--action-script", str(HOOK), "--base-url", BASE_URL],
cwd=ROOT, capture_output=True, env=env)
print((result.stdout + result.stderr).decode("utf-8", "replace").strip())
check(result.returncode == 0, "public failover test failed")
print("api containers after")
print(ps())
stopped = [container for container in containers if container not in api_containers()]
check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}")
print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); "
f"other api containers kept running, postgres/redis/lb were not touched")
stopped_once.add(stopped[0])
docker("start", stopped[0])
wait_balanced(len(containers))
print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy")
if stopped_once == set(containers):
break
check(stopped_once == set(containers), "not every replica was stopped by the public test")
print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored")
if __name__ == "__main__":
main()
+47
View File
@@ -0,0 +1,47 @@
import os
import subprocess
import sys
import time
from common import BASE_URL, ROOT, check, compose_output, docker
HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh")
def started():
containers = compose_output("ps", "-q").split()
lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines()
return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)}
def main():
before = started()
print("before: container, ID, started at")
for name, (cid, at) in sorted(before.items()):
print(f" {name:<28} {cid[:12]} {at}")
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
begin = time.monotonic()
result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence",
"--action-script", str(HOOK), "--base-url", BASE_URL],
cwd=ROOT, capture_output=True, env=env)
seconds = time.monotonic() - begin
print((result.stdout + result.stderr).decode("utf-8", "replace").strip())
check(result.returncode == 0, "public persistence test failed")
after = started()
print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?")
restarted = []
for name, (cid, at) in sorted(after.items()):
same_id = before.get(name, ("", ""))[0] == cid
new_start = before.get(name, ("", ""))[1] != at
print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}")
check(same_id, f"{name}: container was recreated, volumes/IDs must stay")
if new_start:
restarted.append(name.split("-")[-2])
for service in ("postgres", "redis", "s3", "api", "janitor", "lb"):
check(service in restarted, f"{service} was not restarted by the hook")
print("PASS: the hook restarted postgres, redis, s3 (SeaweedFS), all api replicas, janitor and lb (same containers, "
"new start time); data, the attachment bytes and the old token survived")
if __name__ == "__main__":
main()
+258
View File
@@ -0,0 +1,258 @@
import base64
import datetime as dt
import uuid
from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send
def parse_time(value):
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
def test_duplicate_username():
body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD}
call("POST", "/api/v1/users", 201, body=body)
call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user"))
print("PASS: duplicate username returns 409")
def test_invalid_json():
alice = new_user("alice")
chat = new_chat(alice)
call("POST", "/api/v1/users", 400, raw=b'{"username":')
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json",
headers={"Idempotency-Key": new_key()})
print("PASS: invalid JSON returns 400")
def test_wrong_fields():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD})
call("POST", "/api/v1/users", 400,
body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True})
# Отправителя нельзя подставить: он берётся из сессии.
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()})
print("PASS: wrong and extra fields return 400")
def test_foreign_chat():
alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve")
chat = new_chat(alice, bob)
call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"])
call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"])
send(eve, chat["id"], "not my chat", expected=404)
# Несуществующий чат выглядит так же, как чужой.
call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"])
print("PASS: foreign chat returns 404")
def test_empty_message():
alice = new_user("alice")
chat = new_chat(alice)
for text in ("", " ", "\n\t"):
send(alice, chat["id"], text, expected=400)
send(alice, chat["id"], "я" * 4001, expected=400)
send(alice, chat["id"], "я" * 4000)
print("PASS: empty and too long messages return 400, 4000 chars is ok")
def test_pagination():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)]
history, pages = read_history(bob, chat["id"], limit=2)
check(pages == 4, f"expected 4 pages, got {pages}")
check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages")
other = new_chat(alice)
for i in range(3):
send(alice, other["id"], f"Other {i}")
first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json
call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"])
call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"])
print("PASS: pagination works across multiple pages, foreign cursor returns 400")
def test_empty_cursor():
alice = new_user("alice")
chat = new_chat(alice)
send(alice, chat["id"], "one message")
for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"):
for query in ("?cursor=", "?cursor", "?limit=10&cursor="):
call("GET", path + query, 400, token=alice["token"])
call("GET", path, 200, token=alice["token"])
print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page")
def test_chat_list_pagination():
alice, bob = new_user("alice"), new_user("bob")
created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)]
seen, cursor = [], None
while True:
query = "?limit=1" + (f"&cursor={cursor}" if cursor else "")
page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json
seen += [chat["id"] for chat in page["items"]]
cursor = page["next_cursor"]
if cursor is None:
break
check(seen == created, "chat list pages are wrong")
print("PASS: chat list pagination")
def test_unicode():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob, title="Чат 👋")
text = "Привет 👋 你好"
check(send(alice, chat["id"], text).json["text"] == text, "text changed on write")
history, _ = read_history(bob, chat["id"])
check(history[-1]["text"] == text, "text changed on read")
print("PASS: unicode text is stored as is")
def test_nul_character_is_stored():
name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world"
body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"}
user = call("POST", "/api/v1/users", 201, body=body).json
check(user["display_name"] == name, "display_name with U+0000 changed")
token = login(body["username"], body["password"]).json["token"]
check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read")
chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json
check(chat["title"] == title, "title with U+0000 changed")
check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read")
sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text},
headers={"Idempotency-Key": new_key()}).json
history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"]
check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed")
login("nul\u0000user", PASSWORD, expected=401)
print("PASS: strings with U+0000 are stored and returned as is")
def test_user_has_no_password():
body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD}
user = call("POST", "/api/v1/users", 201, body=body).json
check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}")
print("PASS: user response has no password or hash")
def test_password_rules():
def register(password, expected):
body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password}
call("POST", "/api/v1/users", expected, body=body)
register("x" * 11, 400)
register("x" * 129, 400)
register(" " * 12, 400)
register("x" * 12, 201)
register("я" * 128, 201)
print("PASS: password length 12..128, blank password is rejected")
def test_login_gives_token():
username = "lg_" + uuid.uuid4().hex[:10]
user = call("POST", "/api/v1/users", 201,
body={"username": username, "display_name": "Login", "password": PASSWORD}).json
reply = login(username, PASSWORD)
session = reply.json
check(session["token_type"] == "Bearer", "token_type must be Bearer")
check(session["user"] == user, "session user differs from registered user")
check(len(session["token"]) >= 43, "token is too short")
check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque")
check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached")
left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds()
check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s")
me = call("GET", "/api/v1/users/me", token=session["token"]).json
check(me == user, "/users/me returned another user")
print("PASS: Basic login returns opaque Bearer token with expiry")
def test_wrong_password():
user = new_user("wp")
wrong = login(user["username"], "Wrong-password-000", expected=401)
unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401)
check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic")
check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames")
print("PASS: wrong password and unknown user both return the same 401")
def test_malformed_basic():
no_colon = base64.b64encode(b"just_username").decode()
for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"):
headers = {} if header is None else {"Authorization": header}
call("POST", "/api/v1/auth/sessions", 401, headers=headers)
print("PASS: malformed Basic header returns 401")
def test_password_with_colon_and_unicode():
password = "пароль:с:двоеточием 👋"
user = new_user("colon", password=password)
call("GET", "/api/v1/users/me", token=user["token"])
print("PASS: password with colon and unicode works")
def test_bearer_required():
user = new_user("br")
for headers in ({}, {"Authorization": basic(user["username"], user["password"])},
{"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43},
{"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}):
reply = call("GET", "/api/v1/users/me", 401, headers=headers)
check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer")
call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]})
print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401")
def test_x_user_id_ignored():
alice, bob = new_user("alice"), new_user("bob")
me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json
check(me["id"] == alice["id"], "X-User-Id changed the current user")
print("PASS: X-User-Id does not change the current user")
def test_auth_before_body():
alice = new_user("alice")
chat = new_chat(alice)
call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json")
call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401)
print("PASS: 401 comes before 400 and 404")
def test_logout():
user = new_user("lo")
second = login(user["username"], user["password"]).json["token"]
call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"])
call("GET", "/api/v1/users/me", 401, token=user["token"])
call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"])
call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива
again = login(user["username"], user["password"]).json["token"]
check(again != user["token"], "new login must give a new token")
call("GET", "/api/v1/users/me", 200, token=again)
print("PASS: logout revokes only the current session")
if __name__ == "__main__":
run_tests([
test_duplicate_username,
test_invalid_json,
test_wrong_fields,
test_foreign_chat,
test_empty_message,
test_pagination,
test_empty_cursor,
test_chat_list_pagination,
test_unicode,
test_nul_character_is_stored,
test_user_has_no_password,
test_password_rules,
test_login_gives_token,
test_wrong_password,
test_malformed_basic,
test_password_with_colon_and_unicode,
test_bearer_required,
test_x_user_id_ignored,
test_auth_before_body,
test_logout,
])
+268
View File
@@ -0,0 +1,268 @@
"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ.
Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа:
пользовательская ошибка не должна становиться 500.
"""
import hashlib
import http.client
import socket
import urllib.parse
import uuid
import zlib
import images
from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload
URL = urllib.parse.urlsplit(BASE_URL)
def setup():
alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve")
return alice, bob, eve, new_chat(alice, bob)
def sha(data):
return hashlib.sha256(data).hexdigest()
def check_error(reply, code):
check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}")
def test_png_and_jpeg_roundtrip():
alice, bob, _, chat = setup()
for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"),
(images.sample_jpeg(), "image/jpeg", "sample.jpg"),
(images.progressive_jpeg(), "image/jpeg", "progressive.jpg"),
(images.png(33, 17, interlace=1), "image/png", "interlaced.png"),
(images.png(1, 1, gray=True), "image/png", "1x1.png")):
att = upload(alice, chat["id"], data, ctype, name).json
check(att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None, "not ready")
check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ")
check(att["content_type"] == ctype and att["filename"] == name, "metadata differs")
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json
check(meta == att, "GET metadata differs from POST response")
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
check(got.data == data, "downloaded bytes differ")
check(got.headers["content-type"] == ctype, "wrong Content-Type on download")
check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length")
check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing")
print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes")
print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte")
def test_size_limit_exact_boundary():
alice, _, _, chat = setup()
exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES)
check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB")
att = upload(alice, chat["id"], exact).json
check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed")
print(f"exactly {len(exact)} bytes (10 MiB) -> 201")
over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1)
reply = upload(alice, chat["id"], over, expected=413)
check_error(reply, "payload_too_large")
print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body")
print("PASS: the 10 MiB limit is exact")
def chunked_upload(user, chat_id, total, piece=64 * 1024):
"""Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении."""
conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60)
def body():
sent = 0
head = images.sample_png()[:64]
yield head
sent += len(head)
while sent < total:
part = bytes(min(piece, total - sent))
sent += len(part)
yield part
try:
conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(),
headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]},
encode_chunked=True)
reply = conn.getresponse()
return reply.status, reply.read()
finally:
conn.close()
def test_size_limit_streaming():
alice, _, _, chat = setup()
status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024)
check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}")
print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed")
call("GET", "/health/ready", 200)
print("PASS: oversized streamed body is cut off, the API keeps working")
def test_pixel_limit():
alice, _, _, chat = setup()
at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей
att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json
print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 201 {att['id'][:8]}")
over = images.png(8001, 5000, gray=True)
check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large")
print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413")
bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01")
check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large")
print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded")
print("PASS: pixel limit is checked before decompression")
def test_damaged_images():
alice, _, _, chat = setup()
png, jpeg = images.sample_png(), images.sample_jpeg()
cases = [
("PNG cut in half", png[:len(png) // 2], "image/png"),
("PNG without IEND", png[:-12], "image/png"),
("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"),
("PNG with garbage after IEND", png + b"garbage", "image/png"),
("PNG signature only", png[:8], "image/png"),
("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"),
("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"),
("JPEG header only", jpeg[:20], "image/jpeg"),
("random bytes", uuid.uuid4().bytes * 100, "image/png"),
("plain text", "это не картинка".encode(), "image/jpeg"),
]
for name, data, ctype in cases:
reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415)
check_error(reply, "unsupported_media_type")
print(f"{name} ({len(data)} bytes, {ctype}) -> 415")
check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request")
print("empty body -> 400")
print("PASS: damaged and foreign files are rejected with 415, never 500")
def test_mime_substitution():
alice, _, _, chat = setup()
cases = [
("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"),
("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"),
("GIF bytes sent as image/png", images.GIF, "image/png"),
("PNG sent as image/gif", images.sample_png(), "image/gif"),
("PNG sent as text/html", images.sample_png(), "text/html"),
("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"),
("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"),
]
for name, data, ctype in cases:
check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type")
print(f"{name} -> 415")
att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json
check(att["content_type"] == "image/png", "content type must be normalised")
print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png")
print("PASS: the type is taken from the content; a declared type that does not match is refused")
def test_filenames():
alice, _, _, chat = setup()
bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png",
"bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256]
for name in bad:
check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request")
print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400")
check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request")
print("no filename -> 400")
for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"):
att = upload(alice, chat["id"], images.sample_png(), filename=name).json
check(att["filename"] == name, "display name changed")
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"])
check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded")
print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only")
print("PASS: filename never becomes a path; dangerous names are refused")
def test_access_control():
alice, bob, eve, chat = setup()
att = upload(alice, chat["id"], images.sample_png()).json
for suffix in ("", "/content"):
reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
check_error(reply, "not_found")
call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400")
check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found")
check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found")
upload(None, chat["id"], images.sample_png(), expected=401)
print("upload into someone else's / unknown chat -> 404, without token -> 401")
call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", 400, token=bob["token"])
call("GET", f"/api/v1/attachments/{att['id']}/content?variant=original", 200, token=bob["token"])
print("variant=thumbnail -> 400 (not in the lab4 contract), variant=original -> 200")
print("PASS: only chat members can see an attachment")
def test_message_attachments():
alice, bob, eve, chat = setup()
first = upload(alice, chat["id"], images.sample_png()).json
second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json
msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json
check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed")
print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member")
plain = post_message(bob, chat["id"], "без вложений").json
check(plain["attachment_ids"] == [], "default attachment_ids must be []")
reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json
history, _ = read_history(bob, chat["id"])
check(history == [msg, plain, reused], "history differs from the confirmed messages")
print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed")
other_chat = new_chat(alice, eve)
foreign = upload(alice, other_chat["id"], images.sample_png()).json
for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"),
([first["id"], foreign["id"]], "own + foreign attachment"),
([str(uuid.uuid4())], "unknown attachment_id")):
check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found")
print(f"{what} -> 404")
check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found")
print("outsider referencing an attachment of the chat -> 404")
for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"),
([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"),
(["not-a-uuid"], "x", "bad UUID"),
([], "", "empty text without attachments"),
([], " ", "whitespace text without attachments")):
check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request")
print(f"{what} -> 400")
history_after, _ = read_history(bob, chat["id"])
check(history_after == history, "a rejected message was stored")
print("PASS: only attachments of the same chat can be linked; rejected requests change nothing")
def test_idempotency_with_attachments():
alice, bob, _, chat = setup()
att = upload(alice, chat["id"], images.sample_png()).json
other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json
key = new_key()
first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json
again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json
check(again == first, "replay returned a different message")
check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict")
check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict")
key2 = new_key()
plain = post_message(alice, chat["id"], "текст", None, key=key2).json
check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field")
history, _ = read_history(bob, chat["id"])
check(len(history) == 2, "replays created extra messages")
print("same key + same attachments -> same 201; other attachments -> 409; missing field == []")
print("PASS: Idempotency-Key covers attachment_ids")
def test_aborted_upload():
alice, _, _, chat = setup()
sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10)
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n"
f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n")
sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000))
sock.close() # клиент пропал на середине тела
print("sent 1.1 MB of a declared 5 MB body and closed the connection")
att = upload(alice, chat["id"], images.sample_png()).json
call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"])
print("the API keeps serving uploads; nothing of the aborted request is visible "
"(DB/S3 side is checked by test_consistency.py)")
print("PASS: an interrupted upload is dropped")
if __name__ == "__main__":
run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming,
test_pixel_limit, test_damaged_images, test_mime_substitution, test_filenames, test_access_control,
test_message_attachments, test_idempotency_with_attachments, test_aborted_upload])
+103
View File
@@ -0,0 +1,103 @@
from collections import Counter
import images
from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload
def test_requests_spread_over_replicas():
alice = new_user("alice")
replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)]
served = Counter(reply.instance for reply in replies)
print(f"20 x GET /users/me with one token -> replicas: {dict(served)}")
check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned")
check(len(served) >= 2, "all requests went to one replica: balancing does not work")
print("PASS: one session works on every replica, no second login (sessions are shared in Redis)")
def test_write_on_one_read_on_other():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
sent = send(alice, chat["id"], "written on one replica")
readers = set()
for _ in range(6):
reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"])
check(sent.json in reply.json["items"], "confirmed message is not visible on another replica")
readers.add(reply.instance)
print(f"message written by {sent.instance}; read back by {sorted(readers)}")
check(readers - {sent.instance}, "no read went to another replica")
print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)")
def test_logout_seen_by_all_replicas():
user = new_user("logout")
out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"])
after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)]
print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}")
check(len(set(after)) >= 2, "only one replica answered")
print("PASS: logout on one replica revokes the session on all of them")
def test_idempotent_retry_on_other_replica():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
key = new_key()
first = send(alice, chat["id"], "retry me", key=key)
for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику
retry = send(alice, chat["id"], "retry me", key=key)
if retry.instance != first.instance:
break
print(f"first POST on {first.instance} -> 201 {first.json['id']}; "
f"same key on {retry.instance} -> {retry.status} {retry.json['id']}")
check(retry.instance != first.instance, "retry never reached another replica")
check(retry.json == first.json, "retry on another replica returned a different message")
items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"]
check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate")
print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message")
def test_attachment_on_other_replica():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
data = images.sample_png()
sent = upload(alice, chat["id"], data)
readers = set()
for _ in range(6):
reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"])
check(reply.data == data, "another replica returned different bytes")
readers.add(reply.instance)
print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}")
check(readers - {sent.instance}, "no download went to another replica")
print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)")
def test_instance_header_everywhere():
containers = api_containers()
alice, eve = new_user("alice"), new_user("eve")
chat = new_chat(alice)
replies = [
call("GET", "/health/live", 200),
call("GET", "/health/ready", 200),
send(alice, chat["id"], "hello"),
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}),
call("GET", "/api/v1/users/me", 401),
call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]),
call("POST", "/api/v1/users", 409,
body={"username": alice["username"], "display_name": "x", "password": alice["password"]}),
]
for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер
check(any(container.startswith(reply.instance) for container in containers),
f"X-Instance-Id {reply.instance} is not a running api container")
print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container "
f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}")
print("PASS: X-Instance-Id is in every response and points to a real container")
if __name__ == "__main__":
run_tests([
test_requests_spread_over_replicas,
test_write_on_one_read_on_other,
test_logout_seen_by_all_replicas,
test_idempotent_retry_on_other_replica,
test_attachment_on_other_replica,
test_instance_header_everywhere,
])
+275
View File
@@ -0,0 +1,275 @@
"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка).
Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`),
выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает
janitor вручную. Данные синтетические, volumes не удаляются.
python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]
"""
import json
import socket
import sys
import time
import urllib.parse
import images
from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat,
new_user, post_message, run_tests, sql_value, upload, wait_ready)
URL = urllib.parse.urlsplit(BASE_URL)
# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но finish_upload
# (UPDATE ... state='ready' + COMMIT) падает — как если бы реплика умерла сразу после PUT в S3.
CRASH_AFTER_PUT = r'''
import json, struct, sys, tempfile, zlib
from uuid import UUID
import attachments, db, storage, uploads
chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2])
def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d))
data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b""))
f = tempfile.SpooledTemporaryFile(); f.write(data)
import hashlib
upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest())
image = uploads.inspect(upload, "image/png")
created = {}
real_intent = db.create_upload_intent
def remember(attachment_id, *args):
created["id"], created["key"] = str(attachment_id), args[-1]
return real_intent(attachment_id, *args)
def crash(attachment_id):
raise RuntimeError("simulated crash between S3 PUT and DB COMMIT")
db.create_upload_intent = remember
db.finish_upload = crash
db.pool.open()
try:
attachments.store(chat_id, owner_id, "crash.png", upload, image)
outcome = "stored"
except RuntimeError as exc:
outcome = str(exc)
print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"],
"object_exists": storage.exists(created["key"])}))
'''
OBJECT_EXISTS = r'''
import json, sys, storage
print(json.dumps({"exists": storage.exists(sys.argv[1])}))
'''
PUT_ORPHAN = r'''
import hashlib, io, json, uuid, storage
data = b"orphan object without a row in PostgreSQL"
key = storage.new_key(uuid.uuid4().hex)
storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest())
print(json.dumps({"key": key}))
'''
DELETE_OBJECT = r'''
import json, sys, storage
storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]}))
'''
COUNT_OBJECTS = r'''
import json, storage
print(json.dumps({"objects": sum(1 for _ in storage.list_objects())}))
'''
def last_json(output):
return json.loads(output.splitlines()[-1])
def object_exists(key):
return last_json(app_python(OBJECT_EXISTS, key))["exists"]
def backdate(where):
sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}")
def uploading_rows(chat_id):
return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'"))
def wait_download(user, att, seconds=60):
deadline = time.monotonic() + seconds
while True:
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"])
if reply.status == 200:
return reply
check(time.monotonic() < deadline, "download did not recover after S3 start")
time.sleep(1)
def s3_outage(stop_action, start_action, label):
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
data = images.sample_png()
att = upload(alice, chat["id"], data).json
before = uploading_rows(chat["id"])
print(f"--- S3 {label}: docker compose {stop_action} s3")
compose(stop_action, "s3")
try:
started = time.monotonic()
reply = upload(alice, chat["id"], data, expected=503)
ms = (time.monotonic() - started) * 1000
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
body = reply.data.decode().lower()
check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")),
"error leaks storage details")
print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body")
reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
print("download of an existing file -> 503")
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json
check(meta["sha256"] == att["sha256"], "metadata changed")
post_message(alice, chat["id"], "текст без файла")
post_message(alice, chat["id"], "", [att["id"]])
call("GET", "/health/ready", 200)
print("metadata, messages (also with an already stored attachment) and /health/ready keep working: "
"S3 is needed only for file bytes")
after = uploading_rows(chat["id"])
print(f"uploading rows of this chat: before {before}, after the failed upload {after} "
"(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API")
finally:
print(f"docker compose {start_action} s3")
compose(start_action, "s3")
got = wait_download(bob, att)
check(got.data == data, "file changed after S3 recovery")
print("after S3 recovery the old file downloads byte for byte")
upload(alice, chat["id"], data)
print("new upload -> 201")
leftover = uploading_rows(chat["id"])
if leftover:
backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'")
report = janitor()
print(f"janitor after grace period: {json.dumps(report)}")
check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain")
print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up")
def test_s3_stopped():
s3_outage("stop", "start", "stopped (connection refused / name not resolved)")
def test_s3_frozen():
s3_outage("pause", "unpause", "frozen (no answer, timeouts)")
def test_crash_between_s3_and_db():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
ready = upload(alice, chat["id"], images.sample_png()).json
result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"]))
check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}")
att_id, key = result["attachment_id"], result["object_key"]
state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'")
check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3")
print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3")
call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"])
call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"])
post_message(alice, chat["id"], "x", [att_id], expected=404)
print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404")
report = janitor()
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key),
"fresh upload was deleted inside the grace period")
print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> "
"fresh row and object are kept (the upload might still be running)")
backdate(f"id = '{att_id}'")
report = janitor("--dry-run")
check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report")
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row")
print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted")
report = janitor()
check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload")
check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists")
check(not object_exists(key), "object still exists")
print(f"janitor: {json.dumps(report)} -> row and object deleted")
report = janitor()
print(f"janitor again: {json.dumps(report)} -> idempotent")
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
check(got.data == images.sample_png(), "ready attachment damaged by cleanup")
print("the ready attachment of the same chat is untouched")
print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period")
def test_orphan_objects():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
ready = upload(alice, chat["id"], images.sample_png()).json
key = last_json(app_python(PUT_ORPHAN))["key"]
print(f"object {key} put into the bucket without a row in PostgreSQL")
# Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с.
short = {"UPLOAD_TIMEOUT_SECONDS": "1"}
report = janitor("--grace-seconds", "3", env=short)
check(object_exists(key), "fresh orphan deleted inside grace period")
print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept")
time.sleep(4)
report = janitor("--grace-seconds", "3", "--dry-run", env=short)
check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key),
"dry run must report and keep the orphan")
print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept")
report = janitor("--grace-seconds", "3", env=short)
check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted")
print(f"janitor: {json.dumps(report)} -> orphan deleted")
got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"])
check(got.data == images.sample_png(), "ready attachment was deleted with orphans")
print("ready attachments older than the grace period are untouched (they have rows)")
victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json
victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'")
app_python(DELETE_OBJECT, victim_key)
report = janitor("--grace-seconds", "3", env=short)
check(report["missing_objects"] >= 1, "missing object not reported")
check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted")
reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"])
print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects="
f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)")
print("PASS: orphans are found and deleted only after the grace period; missing objects are reported")
def api_tmp_files():
counts = {}
for container in compose_output("ps", "-q", "api").split():
counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0)
return counts
def test_aborted_upload_leaves_nothing():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
objects = last_json(app_python(COUNT_OBJECTS))["objects"]
rows = int(sql_value("SELECT count(*) FROM attachments"))
for sent in (100_000, 3_000_000):
sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10)
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n"
f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n"
f"Content-Length: 9000000\r\n\r\n")
sock.sendall(head.encode() + images.sample_png() + bytes(sent))
sock.close()
print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection")
time.sleep(2)
check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row")
check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object")
print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}")
tmp = api_tmp_files()
check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}")
print(f"temporary upload files in api containers: {tmp} (removed when the request ends)")
for container in compose_output("ps", "-q", "api").split():
mounts = docker("inspect", "-f", "{{json .Mounts}}", container)
# файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных
storage = [m for m in json.loads(mounts)
if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)]
check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}")
print("api containers have no volumes: the original is never stored locally")
print("PASS: an interrupted upload leaves no row, no object and no local file")
if __name__ == "__main__":
target = sys.argv[1] if len(sys.argv) > 1 else "all"
groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db],
"orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]}
groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"]
check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]")
wait_ready()
run_tests(groups[target])
+101
View File
@@ -0,0 +1,101 @@
import time
import uuid
from common import call, check, compose_output, new_user, psql, run_tests
def sql_value(sql):
code, output, error = psql(sql)
check(code == 0, f"psql failed: {error}")
return output
def test_create_chat_rolls_back():
alice = new_user("rollback")
title = "rollback-" + uuid.uuid4().hex[:8]
inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
call("POST", "/api/v1/chats", 400, token=alice["token"],
body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]})
left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'")
orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS "
"(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)")
inserted_after = inserted_before
deadline = time.monotonic() + 15
while inserted_after == inserted_before and time.monotonic() < deadline:
time.sleep(0.5)
inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; "
f"chats insert counter: {inserted_before} -> {inserted_after}")
check(left == "0" and orphans == "0", "chat row survived the failed transaction")
check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown")
print("PASS: chat row was inserted and then rolled back together with members")
def test_message_needs_membership_in_db():
code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) "
f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))")
print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}")
check(code != 0 and "foreign key" in error, "database accepted a message from a non-member")
print("PASS: the database itself rejects a message from a non-member")
def test_password_is_argon2id():
user = new_user("hash")
stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'")
print(f"stored password: {stored[:31]}... (length {len(stored)})")
check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash")
print("PASS: password is stored as Argon2id hash")
def test_unique_username_constraint():
constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint "
"WHERE conrelid = 'users'::regclass AND contype = 'u'")
print(f"unique constraint on users: {constraint}")
check(constraint == "UNIQUE (username)", "no unique constraint on username")
print("PASS: username is unique in the database")
def test_migrations_are_not_repeated():
versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations")
print(f"applied migrations: {versions}; running the migrate service once more")
output = compose_output("run", "--rm", "--no-deps", "-T", "migrate")
again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations")
last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1]
print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}")
check(again == versions and "none" in last_log, "migrations were applied again")
print("PASS: migrations are applied once, a repeated run does nothing")
def test_replicas_do_not_migrate():
"""lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему."""
logs = compose_output("logs", "api").splitlines()
checked = [line for line in logs if "Schema is up to date" in line]
migrated = [line for line in logs if "Migrations applied" in line]
migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip()
print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': "
f"{len(migrated)}; migrate service: {migrate_exit}")
check(checked and not migrated, "api replicas must not apply migrations")
check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0")
print("PASS: only the migrate service applies migrations, replicas wait for the schema")
def test_idempotency_key_is_unique_in_db():
constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint "
"WHERE conname = 'messages_idempotency_key'")
print(f"unique constraint for Idempotency-Key: {constraint}")
check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys")
print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database")
if __name__ == "__main__":
run_tests([
test_create_chat_rolls_back,
test_message_needs_membership_in_db,
test_password_is_argon2id,
test_unique_username_constraint,
test_migrations_are_not_repeated,
test_replicas_do_not_migrate,
test_idempotency_key_is_unique_in_db,
])
+57
View File
@@ -0,0 +1,57 @@
import datetime as dt
import os
import time
from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready
SHORT_TTL = 2
def restart_api(ttl=None):
env = dict(os.environ)
if ttl is None:
env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env
else:
env["SESSION_TTL_SECONDS"] = str(ttl)
compose("up", "-d", "--no-deps", "--wait", "api", env=env)
wait_ready()
def seconds_left(expires_at):
moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00"))
return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds()
def test_session_expires():
print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}")
restart_api(SHORT_TTL)
try:
user = new_user("ttl")
session = login(user["username"], user["password"]).json
token = session["token"]
left = seconds_left(session["expires_at"])
print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: "
f"{redis_cli('pttl', session_key(token))} ms")
check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions")
call("GET", "/api/v1/users/me", 200, token=token)
print("before expiry: GET /users/me -> 200")
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0)
call("GET", "/api/v1/users/me", 401, token=token)
exists = redis_cli("exists", session_key(token))
print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}")
check(exists == "0", "Redis must delete the expired session key")
finally:
print("recreating api with SESSION_TTL_SECONDS from .env")
restart_api()
user = new_user("ttl_back")
left = seconds_left(login(user["username"], user["password"]).json["expires_at"])
print(f"normal API again: new session expires in {left:.0f} s")
check(left > 300, "normal TTL was not restored")
print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401")
if __name__ == "__main__":
run_tests([test_session_expires])
+110
View File
@@ -0,0 +1,110 @@
import sys
import time
import uuid
from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready
FAKE_TOKEN = "A" * 43
def timed(method, path, expected, **kwargs):
started = time.monotonic()
reply = call(method, path, expected, **kwargs)
return reply, (time.monotonic() - started) * 1000
def prepare():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
before = send(alice, chat["id"], "confirmed before outage").json
return alice, bob, chat, before
def recovered(alice, chat, before):
wait_ready()
history, _ = read_history(alice, chat["id"])
check(before in history, "message confirmed before the outage is lost or changed")
call("GET", "/api/v1/users/me", 200, token=alice["token"])
print("after start: the old token works, confirmed messages are in place")
def test_redis_down():
alice, bob, chat, before = prepare()
print("stopping redis")
compose("stop", "redis")
try:
call("GET", "/health/live", 200)
_, ms = timed("GET", "/health/ready", 503)
print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)")
reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"])
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked")
call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN)
print("unknown token -> 503, not 200: nobody is trusted without a check")
call("GET", "/api/v1/chats", 401)
print("no token -> 401")
call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)})
print("login -> 503: there is nowhere to store the session")
call("POST", "/api/v1/users", 201,
body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD})
print("registration -> 201: it needs only PostgreSQL")
finally:
print("starting redis")
compose("start", "redis")
recovered(alice, chat, before)
print("PASS: Redis outage gives 503 and does not let anyone in without a session check")
def test_redis_crash():
alice = new_user("crash")
print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown")
compose("kill", "redis")
try:
# Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy.
status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status
print(f"right after the crash: GET /users/me -> {status}")
finally:
print("starting redis")
compose("start", "redis")
wait_ready()
call("GET", "/api/v1/users/me", 200, token=alice["token"])
print("after start: the session issued before the crash still works (AOF, appendfsync always)")
print("PASS: sessions survive a Redis crash, not only a clean restart")
def test_postgres_down():
alice, bob, chat, before = prepare()
print("stopping postgres")
compose("stop", "postgres")
try:
call("GET", "/health/live", 200)
_, ms = timed("GET", "/health/ready", 503)
print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)")
for method, path, body in (("GET", "/api/v1/users/me", None),
("GET", "/api/v1/chats", None),
("GET", f"/api/v1/chats/{chat['id']}/messages", None),
("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})):
reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()})
check(reply.json["error"]["code"] == "unavailable", "expected code unavailable")
check("postgres" not in reply.data.decode().lower(), "error must not show database details")
print("valid token: users/me, chats, history, new message -> 503 unavailable, no details")
call("GET", "/api/v1/chats", 401)
call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN)
print("no token -> 401, unknown token -> 401: anonymous users are not let in")
call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)})
call("POST", "/api/v1/users", 503,
body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD})
print("login -> 503, registration -> 503")
finally:
print("starting postgres")
compose("start", "postgres")
recovered(alice, chat, before)
print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out")
if __name__ == "__main__":
target = sys.argv[1] if len(sys.argv) > 1 else "all"
tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down],
"all": [test_redis_down, test_redis_crash, test_postgres_down]}
check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]")
run_tests(tests[target])
+174
View File
@@ -0,0 +1,174 @@
import json
import socket
import threading
import time
import urllib.parse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send
PARALLEL = 20
def post_raw(user, chat_id, raw, key):
"""POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы."""
return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"],
raw=raw, headers={"Idempotency-Key": key})
def texts_in(user, chat_id):
history, _ = read_history(user, chat_id)
return [message["text"] for message in history]
def test_replay_returns_same_message():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
key = new_key()
first = send(alice, chat["id"], "Привет 👋", key=key)
# То же JSON-значение, записанное иначе: пробелы и \u-экранирование.
same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key)
again = send(alice, chat["id"], "Привет 👋", key=key)
print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}")
check(same.status == 201 and same.json == first.json and again.json == first.json,
"retry must return the original 201 and the same message")
check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate")
print("PASS: a retry with the same key returns the original 201 and message, no duplicate")
def test_same_key_parallel_through_replicas():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
key = new_key()
barrier = threading.Barrier(PARALLEL)
def post(_):
barrier.wait() # все запросы стартуют одновременно
return send(alice, chat["id"], "one message for 20 requests", key=key)
with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
replies = list(pool.map(post, range(PARALLEL)))
bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies}
served = Counter(reply.instance for reply in replies)
count = texts_in(bob, chat["id"]).count("one message for 20 requests")
print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, "
f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}")
check(len(served) >= 2, "requests did not go through different replicas")
check(len(bodies) == 1 and count == 1, "parallel retries created more than one message")
print("PASS: parallel requests with one key through different replicas create exactly one message")
def test_same_key_different_bodies_parallel():
alice = new_user("alice")
chat = new_chat(alice)
key = new_key()
barrier = threading.Barrier(PARALLEL)
def post(index):
text = "body A" if index % 2 == 0 else "body B"
barrier.wait()
reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"],
body={"text": text}, headers={"Idempotency-Key": key})
return text, reply
with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
results = list(pool.map(post, range(PARALLEL)))
texts = texts_in(alice, chat["id"])
winner = [t for t in texts if t in ("body A", "body B")]
print(f"{PARALLEL} parallel POSTs, one key, two different bodies: "
f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}")
check(len(winner) == 1, "exactly one message must be created")
for text, reply in results:
expected = 201 if text == winner[0] else 409
check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}")
print("PASS: the first body wins, the same body gets 201, the other body gets 409")
def test_key_scope():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
other = new_chat(alice, bob)
key = new_key()
a1 = send(alice, chat["id"], "scope", key=key).json
b1 = send(bob, chat["id"], "scope", key=key).json
a2 = send(alice, other["id"], "scope", key=key).json
print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages")
check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat")
call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"],
body={"text": "other text"}, headers={"Idempotency-Key": key})
print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409")
def test_key_validation():
alice = new_user("alice")
chat = new_chat(alice)
path = f"/api/v1/chats/{chat['id']}/messages"
call("POST", path, 400, token=alice["token"], body={"text": "no key"})
for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"):
call("POST", path, 400, token=alice["token"], body={"text": "bad key"},
headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8
longest = ("Az09._:-" * 16)[:128]
call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest})
print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201")
print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}")
def test_failed_request_does_not_burn_key():
alice, eve = new_user("alice"), new_user("eve")
chat = new_chat(alice)
key = new_key()
path = f"/api/v1/chats/{chat['id']}/messages"
call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key})
call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key})
created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key})
print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}")
print("PASS: rejected requests (400, 404) do not store a result for the key")
def test_replay_checks_session_and_access():
alice, eve = new_user("alice"), new_user("eve")
chat = new_chat(alice)
key = new_key()
send(alice, chat["id"], "private", key=key)
stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"],
body={"text": "private"}, headers={"Idempotency-Key": key})
call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"],
body={"text": "private"}, headers={"Idempotency-Key": key})
print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}")
print("PASS: a retry checks the session and chat access again, the saved answer is not leaked")
def test_lost_response_then_retry():
"""Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен."""
alice = new_user("alice")
chat = new_chat(alice)
key = new_key()
body = json.dumps({"text": "answer was lost"}).encode("utf-8")
url = urllib.parse.urlsplit(BASE_URL)
request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n"
f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n"
f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode()
with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection:
connection.sendall(request + body) # и сразу закрываем, ответ не читаем
time.sleep(1)
retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"],
body={"text": "answer was lost"}, headers={"Idempotency-Key": key})
count = texts_in(alice, chat["id"]).count("answer was lost")
print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}")
check(count == 1, "the retry after a lost answer created a duplicate")
print("PASS: an unconfirmed POST can be safely retried with the same key")
if __name__ == "__main__":
run_tests([
test_replay_returns_same_message,
test_same_key_parallel_through_replicas,
test_same_key_different_bodies_parallel,
test_key_scope,
test_key_validation,
test_failed_request_does_not_burn_key,
test_replay_checks_session_and_access,
test_lost_response_then_retry,
])
+57
View File
@@ -0,0 +1,57 @@
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send
CLIENTS = 10
MESSAGES_PER_CLIENT = 10
REGISTRATIONS = 20
def test_concurrent_messages():
users = [new_user(f"race{i}") for i in range(CLIENTS)]
chat = new_chat(*users, title="race")
barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно
def client(index):
barrier.wait()
for n in range(MESSAGES_PER_CLIENT):
send(users[index], chat["id"], f"client-{index}-message-{n}")
started = time.monotonic()
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков
elapsed = time.monotonic() - started
history, pages = read_history(users[0], chat["id"], limit=30)
ids = {message["id"] for message in history}
expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)}
print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; "
f"read {len(history)} in {pages} pages, unique ids: {len(ids)}")
check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages")
check(len(ids) == len(history), "duplicate message ids")
check({message["text"] for message in history} == expected, "lost or changed message texts")
print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates")
def test_concurrent_registration():
username = "race_" + uuid.uuid4().hex[:10]
barrier = threading.Barrier(REGISTRATIONS)
def register(index):
barrier.wait()
body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD}
return call("POST", "/api/v1/users", (201, 409), body=body).status
with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool:
statuses = list(pool.map(register, range(REGISTRATIONS)))
created, conflicts = statuses.count(201), statuses.count(409)
print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}")
check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win")
print("PASS: concurrent registration created exactly one user")
if __name__ == "__main__":
run_tests([test_concurrent_messages, test_concurrent_registration])
+146
View File
@@ -0,0 +1,146 @@
"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен.
Нужен запущенный стенд из compose.yaml.
"""
import json
import socket
import images
from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload
# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API.
ANONYMOUS = r'''
import json, sys, urllib.request, urllib.error
import settings
base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET
def status(method, url, data=None):
try:
with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r:
return r.status, r.read(300).decode("utf-8", "replace")
except urllib.error.HTTPError as e:
return e.code, e.read(300).decode("utf-8", "replace")
out = {}
for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None),
("head_object", "HEAD", base + "/" + sys.argv[1], None),
("list_bucket", "GET", base + "?list-type=2", None),
("put_object", "PUT", base + "/originals/anonymous-test", b"x"),
("get_bucket_policy", "GET", base + "?policy", None)):
code, body = status(method, url, data)
out[name] = {"status": code, "s3_code": body.split("<Code>")[1].split("</Code>")[0] if "<Code>" in body else ""}
print(json.dumps(out))
'''
# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json).
API_KEY_RIGHTS = r'''
import io, json, sys, uuid
from botocore.exceptions import ClientError
import settings, storage
c, bucket = storage._client, settings.S3_BUCKET
def attempt(name, fn):
try:
fn(); return "allowed"
except ClientError as e:
return "denied: " + e.response.get("Error", {}).get("Code", "?")
public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]},
"Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]})
key = "originals/" + uuid.uuid4().hex
result = {
"get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()),
"put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")),
"delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)),
"list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)),
"create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])),
"make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)),
"read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)),
"delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)),
"change versioning": attempt("versioning", lambda: c.put_bucket_versioning(
Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})),
}
try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket
result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", []))
except ClientError as e:
result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?")
print(json.dumps(result))
'''
def last_json(output):
return json.loads(output.splitlines()[-1])
def prepare():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
att = upload(alice, chat["id"], images.sample_png()).json
key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'")
return alice, bob, att, key
def test_bucket_is_private():
alice, bob, att, key = prepare()
result = last_json(app_python(ANONYMOUS, key))
for name, reply in result.items():
print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}")
check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}")
call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
print("the same file through the API for a chat member -> 200")
print("PASS: the bucket is private: no anonymous read, list, write or policy read")
def test_api_key_is_limited():
alice, bob, att, key = prepare()
result = last_json(app_python(API_KEY_RIGHTS, key))
allowed = {"get own object", "put object", "delete object", "list bucket"}
visible = result.pop("visible buckets")
print(f"API key: {'list buckets':<32} -> {visible}")
check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets")
for name, outcome in result.items():
print(f"API key: {name:<32} -> {outcome}")
if name in allowed:
check(outcome == "allowed", f"API key must be allowed to {name}")
else:
check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}")
print("PASS: the API key can only get/put/delete/list objects of its own bucket; "
"bucket admin, policy and other buckets are denied")
def test_no_open_ports_or_console():
config = json.loads(compose_output("config", "--format", "json"))
s3 = config["services"]["s3"]
check(not s3.get("ports"), "s3 must not publish ports")
check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network")
check(config["networks"]["data"].get("internal") is True, "data network must be internal")
print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, "
"filer UI 8888 are reachable only inside the stand)")
for port in (8333, 9333, 8888, 19333):
sock = socket.socket()
sock.settimeout(1)
refused = sock.connect_ex(("127.0.0.1", port)) != 0
sock.close()
check(refused, f"127.0.0.1:{port} is reachable from the host")
print(f"127.0.0.1:{port} from the host -> connection refused")
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()}
for name, items in sorted(secrets.items()):
print(f"secrets of {name}: {items or '-'}")
for name in ("api", "janitor"):
check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"},
f"{name} must not get S3 admin keys or the key config")
check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key")
print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init")
def test_bootstrap_is_idempotent():
alice, bob, att, key = prepare()
output = compose_output("run", "--rm", "--no-deps", "s3-init")
print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip()))
check("bucket is private" in output, "bootstrap did not confirm a private bucket")
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
check(got.data == images.sample_png(), "file changed after repeated bootstrap")
upload(alice, att["chat_id"], images.sample_png())
print("after the repeated bootstrap: old file downloads, new upload works")
print("PASS: bucket/user/policy bootstrap can be repeated safely")
if __name__ == "__main__":
run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console,
test_bootstrap_is_idempotent])
+218
View File
@@ -0,0 +1,218 @@
"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик.
Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что
тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково
(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше
DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование)
при тех же 10 загрузках не помещается в лимит 512 MiB.
Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков
одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status
(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера
(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с.
python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json]
"""
import argparse
import hashlib
import json
import platform
import statistics
import threading
import time
import images
from common import ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_balanced, wait_ready
def proc_status(container):
"""VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты)."""
script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; "
"for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current "
"/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; "
"do [ -r $f ] && echo \"$f $(cat $f)\"; done; true")
out = docker("exec", container, "sh", "-c", script)
values = {}
for line in out.splitlines():
parts = line.replace(":", " ").split()
if parts[0] in ("VmRSS", "VmHWM"):
values[parts[0]] = int(parts[1])
elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")):
values["cgroup_peak"] = int(parts[1])
elif parts[0].endswith(("memory.current", "usage_in_bytes")):
values["cgroup_current"] = int(parts[1])
return values
class Sampler(threading.Thread):
def __init__(self, containers):
super().__init__(daemon=True)
self.containers = containers
self.stop = threading.Event()
self.rss = {c: 0 for c in containers}
self.cgroup = {c: 0 for c in containers}
self.samples = 0
def run(self):
while not self.stop.is_set():
for c in self.containers:
try:
v = proc_status(c)
except Exception: # noqa: BLE001 — один пропущенный замер не важен
continue
self.rss[c] = max(self.rss[c], v.get("VmRSS", 0))
self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0))
self.samples += 1
def api_containers():
from common import api_containers as running
return running()
def fresh_replicas():
compose("restart", "api")
containers = wait_balanced(2, seconds=90)
wait_ready()
return containers
def warm_up(user, chat):
"""По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п."""
for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4:
upload(user, chat["id"], data, ctype)
def scenario(name, payloads, concurrency):
user = new_user("mem")
chat = new_chat(user)
containers = fresh_replicas()
warm_up(user, chat)
before = {c: proc_status(c) for c in containers}
sampler = Sampler(containers)
sampler.start()
barrier = threading.Barrier(concurrency)
results = [None] * concurrency
def worker(i):
data, ctype = payloads[i % len(payloads)]
barrier.wait()
started = time.monotonic()
try:
reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180)
results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data)
except AssertionError as exc:
results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data)
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
wall = time.monotonic()
for t in threads:
t.start()
for t in threads:
t.join()
wall = time.monotonic() - wall
time.sleep(0.5)
sampler.stop.set()
sampler.join()
after = {c: proc_status(c) for c in containers}
ok = [r for r in results if r[0] == 201]
check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 201]}")
for status, _, _, att, data in ok[:3]:
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"])
check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs")
latencies = sorted(r[1] for r in results)
per_replica = []
for c in containers:
b, a = before[c], after[c]
per_replica.append({
"replica": c[:12],
"uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])),
"rss_before_mib": round(b["VmRSS"] / 1024, 1),
"rss_peak_mib": round(a["VmHWM"] / 1024, 1),
"rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1),
"rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1),
"cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None,
"cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None,
})
sizes = sorted({len(d) for d, _ in payloads})
summary = {
"scenario": name,
"concurrency": concurrency,
"file_bytes": sizes,
"total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1),
"all_201": True,
"wall_seconds": round(wall, 2),
"latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)},
"sampler_rounds": sampler.samples,
"replicas": per_replica,
}
print(json.dumps(summary, ensure_ascii=False))
return summary
# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование.
NAIVE_VS_STREAMING = r'''
import json, os, subprocess, sys
code = r"""
import io, os, resource, struct, sys, zlib
sys.path.insert(0, os.getcwd())
os.environ.setdefault("S3_ACCESS_KEY_ID", "x")
from PIL import Image
import uploads # импорт до замера: сравниваем только саму проверку файла
Image.MAX_IMAGE_PIXELS = None
def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d))
w, h = 8000, 5000
comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3)
idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush()
data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
del idat, comp, row
base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
if sys.argv[1] == "naive":
body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB)
im = Image.open(io.BytesIO(data)); im.load()
else:
f = io.BytesIO(data); uploads.check_png(f)
print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024)
"""
out = {}
for mode in ("naive", "streaming"):
r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd())
out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:]
print(json.dumps(out))
'''
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--output", default="evidence/upload-memory.json")
args = parser.parse_args()
check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads")
print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}")
print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...")
large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)]
pixels = [(images.png(8000, 5000), "image/png")]
jpeg = [(images.sample_jpeg(), "image/jpeg")]
print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes")
runs = [
scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency),
scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency),
scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency),
]
naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1])
print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) "
f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB")
limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split()
result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20,
"api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive}
(ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"])
print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB "
f"(limit {result['api_limit_mib']} MiB); results in {args.output}")
if __name__ == "__main__":
main()
+351
View File
@@ -0,0 +1,351 @@
"""Приём и проверка загружаемого изображения с ограниченной памятью.
1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше
MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном
файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса.
2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях
читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково
(CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма),
JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение.
"""
import asyncio
import hashlib
import logging
import struct
import tempfile
import threading
import time
import unicodedata
import zlib
from dataclasses import dataclass
from typing import BinaryIO
from PIL import Image
from starlette.requests import ClientDisconnect, Request
import settings
from errors import bad_request, payload_too_large, unsupported_media_type
log = logging.getLogger("messenger.uploads")
PNG = "image/png"
JPEG = "image/jpeg"
ALLOWED_TYPES = (PNG, JPEG)
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SIGNATURE = b"\xff\xd8\xff"
SPOOL_IN_MEMORY = 256 * 1024
INFLATE_CHUNK = 64 * 1024
CRC_CHUNK = 64 * 1024
# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке).
Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS
# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY.
_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY)
@dataclass
class Upload:
file: BinaryIO
size: int
sha256: str
@dataclass
class ImageInfo:
content_type: str
width: int
height: int
class Damaged(Exception):
pass
class TooManyPixels(Exception):
pass
# ---------- имя файла и заголовки ----------
def validate_filename(value: str | None) -> str:
"""filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов."""
if value is None:
raise bad_request("Query parameter filename is required")
if not 1 <= len(value) <= 255 or not value.strip():
raise bad_request("filename must be 1-255 characters")
if value in (".", ".."):
raise bad_request("filename is not allowed")
for char in value:
if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"):
raise bad_request("filename contains forbidden characters")
return value
def declared_type(header: str | None) -> str:
base = (header or "").split(";", 1)[0].strip().lower()
if base not in ALLOWED_TYPES:
raise unsupported_media_type("Content-Type must be image/png or image/jpeg")
return base
def declared_length(header: str | None) -> int | None:
if header is None:
return None
if not header.isdigit():
raise bad_request("Invalid Content-Length")
length = int(header)
if length > settings.MAX_UPLOAD_BYTES:
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
return length
# ---------- приём тела ----------
async def receive(request: Request) -> Upload:
"""Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется."""
spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR)
digest = hashlib.sha256()
size = 0
deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS
stream = request.stream().__aiter__()
try:
while True:
wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic())
if wait <= 0:
raise bad_request("Upload took too long")
try:
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
except StopAsyncIteration:
break
except asyncio.TimeoutError:
raise bad_request("Upload stalled or took too long") from None
if not chunk:
continue
size += len(chunk)
if size > settings.MAX_UPLOAD_BYTES:
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
digest.update(chunk)
spool.write(chunk)
except ClientDisconnect:
spool.close()
log.info("Upload aborted by the client after %s bytes, nothing stored", size)
raise bad_request("Upload was interrupted") from None
except BaseException:
spool.close()
raise
if size == 0:
spool.close()
raise bad_request("Empty body")
return Upload(spool, size, digest.hexdigest())
async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None:
"""Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв.
Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается.
"""
read = 0
deadline = time.monotonic() + seconds
try:
stream = request.stream().__aiter__()
while read <= limit:
wait = deadline - time.monotonic()
if wait <= 0:
return
try:
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
except (StopAsyncIteration, asyncio.TimeoutError):
return
read += len(chunk)
except (ClientDisconnect, RuntimeError):
return
# ---------- проверка содержимого ----------
def sniff(head: bytes) -> str | None:
if head.startswith(PNG_SIGNATURE):
return PNG
if head.startswith(JPEG_SIGNATURE):
return JPEG
return None
def inspect(upload: Upload, declared: str) -> ImageInfo:
upload.file.seek(0)
detected = sniff(upload.file.read(16))
if detected is None:
raise unsupported_media_type("File is not a PNG or JPEG image")
if detected != declared:
raise unsupported_media_type("Content-Type does not match the file content")
upload.file.seek(0)
try:
with _decode_slots:
width, height = check_png(upload.file) if detected == PNG else check_jpeg(upload.file)
except TooManyPixels:
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
except Damaged as exc:
log.info("Rejected damaged %s: %s", detected, exc)
raise unsupported_media_type("Image is damaged or not supported") from None
except Image.DecompressionBombError:
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc:
log.info("Rejected damaged %s: %s", detected, type(exc).__name__)
raise unsupported_media_type("Image is damaged or not supported") from None
finally:
upload.file.seek(0)
return ImageInfo(detected, width, height)
def check_pixels(width: int, height: int) -> None:
if width < 1 or height < 1:
raise Damaged("zero dimension")
if width * height > settings.MAX_IMAGE_PIXELS:
raise TooManyPixels()
def check_jpeg(file: BinaryIO) -> tuple[int, int]:
with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS
width, height = image.size
check_pixels(width, height)
# Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8:
# 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB.
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
image.load()
return width, height
# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR).
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
FILTER_BYTES = bytes(range(5))
class PngRows:
"""Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки."""
def __init__(self, width, height, depth, color, interlace):
bits = depth * PNG_CHANNELS[color]
def row_bytes(w):
return (w * bits + 7) // 8
passes = [(width, height)] if interlace == 0 else [
((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7]
# (длина строки с байтом фильтра, байт в проходе)
self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0]
self.expected = sum(total for _, total in self.passes)
self.index, self.pos = 0, 0
self.total = 0
def feed(self, data: bytes) -> None:
view = memoryview(data)
self.total += len(data)
if self.total > self.expected:
raise Damaged("more pixel data than the header says")
offset = 0
while offset < len(view):
stride, size = self.passes[self.index]
take = min(len(view) - offset, size - self.pos)
first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride
if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES):
raise Damaged("invalid filter type")
self.pos += take
offset += take
if self.pos == size:
self.index, self.pos = self.index + 1, 0
def done(self) -> bool:
return self.total == self.expected
def check_png(file: BinaryIO) -> tuple[int, int]:
if file.read(8) != PNG_SIGNATURE:
raise Damaged("signature")
rows = inflater = None
width = height = 0
seen_idat = idat_finished = have_plte = False
color = None
first = True
while True:
header = file.read(8)
if len(header) < 8:
raise Damaged("truncated: no IEND")
length, kind = struct.unpack(">I4s", header)
if length > 0x7FFFFFFF:
raise Damaged("chunk length")
if first and kind != b"IHDR":
raise Damaged("IHDR must be first")
crc = zlib.crc32(kind)
keep = kind in (b"IHDR",)
kept = b""
left = length
while left:
piece = file.read(min(CRC_CHUNK, left))
if not piece:
raise Damaged("truncated chunk")
left -= len(piece)
crc = zlib.crc32(piece, crc)
if keep:
kept += piece
elif kind == b"IDAT":
if rows is None or idat_finished:
raise Damaged("IDAT out of order")
feed_inflate(inflater, rows, piece)
stored = file.read(4)
if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF:
raise Damaged(f"CRC mismatch in {kind!r}")
if kind == b"IHDR":
if not first or length != 13:
raise Damaged("IHDR")
width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept)
if width > 0x7FFFFFFF or height > 0x7FFFFFFF:
raise Damaged("dimensions")
check_pixels(width, height) # до распаковки
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]:
raise Damaged("color type / bit depth")
if compression != 0 or filter_method != 0 or interlace not in (0, 1):
raise Damaged("IHDR methods")
rows = PngRows(width, height, depth, color, interlace)
inflater = zlib.decompressobj()
elif kind == b"PLTE":
have_plte = True
elif kind == b"IDAT":
seen_idat = True
elif kind == b"IEND":
break
if seen_idat and kind != b"IDAT":
idat_finished = True
first = False
if file.read(1):
raise Damaged("data after IEND")
if color == 3 and not have_plte:
raise Damaged("palette image without PLTE")
if not seen_idat:
raise Damaged("no IDAT")
tail = inflater.flush(INFLATE_CHUNK)
if tail:
rows.feed(tail)
if not inflater.eof or inflater.unused_data or not rows.done():
raise Damaged("pixel data is incomplete")
return width, height
def feed_inflate(inflater, rows: PngRows, data: bytes) -> None:
"""Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память."""
if inflater.eof:
if data:
raise Damaged("data after the end of zlib stream")
return
out = inflater.decompress(data, INFLATE_CHUNK)
rows.feed(out)
while inflater.unconsumed_tail and not inflater.eof:
out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK)
rows.feed(out)
if inflater.eof and inflater.unused_data:
raise Damaged("data after the end of zlib stream")