forked from mtusi2026/mtusi2026lab4
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e83279ec7f | ||
|
|
0150b91ec0 | ||
|
|
f875e2977f |
@@ -18,3 +18,9 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
research
|
||||
student-tests
|
||||
scripts
|
||||
docs
|
||||
requirements-research.txt
|
||||
*.md
|
||||
|
||||
+19
-12
@@ -1,20 +1,27 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Секреты (пароль БД, DATABASE_URL) создаёт scripts/init-secrets.sh в каталоге secrets/ — не в .env.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Срок жизни сессии; для проверки expiry запускайте с SESSION_TTL_SECONDS=2 (см. student-tests/test_expiry.sh)
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
# TTL записи в кэше Valkey (не больше остатка жизни сессии)
|
||||
SESSION_CACHE_TTL_SECONDS=60
|
||||
# Источник сессий: PostgreSQL (истина) + Valkey (кэш). Адрес кэша задан в compose.yaml (VALKEY_URL).
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
DB_POOL_MAX=10
|
||||
# Файлы секретов внутри контейнера (создаются scripts/init-secrets.sh)
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# --- lab3 ---
|
||||
# Число API-реплик (compose deploy.replicas); также можно: docker compose up -d --scale api=4
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
API_CPUS=1.0
|
||||
API_MEM=256m
|
||||
POSTGRES_CPUS=1.0
|
||||
LB_CPUS=1.0
|
||||
# Пауза между SIGTERM и закрытием listener: LB успевает исключить реплику (rolling update без ошибок)
|
||||
DRAIN_SECONDS=4
|
||||
# --- lab4 ---
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
S3_CPUS=1.0
|
||||
UPLOAD_CONCURRENCY=4
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# Базовый образ закреплён тегом и digest (python 3.12.12, Debian bookworm).
|
||||
ARG PYTHON_IMAGE=python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
|
||||
|
||||
# --- сборка зависимостей в отдельном слое (в runtime не попадают pip и кэш) ---
|
||||
FROM ${PYTHON_IMAGE} AS builder
|
||||
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1
|
||||
WORKDIR /build
|
||||
COPY requirements.txt .
|
||||
RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt
|
||||
|
||||
# --- runtime ---
|
||||
FROM ${PYTHON_IMAGE}
|
||||
ENV PATH=/opt/venv/bin:$PATH \
|
||||
PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/srv/src \
|
||||
APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab LIMIT_CONCURRENCY=1000 MIGRATIONS_DIR=/srv/migrations
|
||||
RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
WORKDIR /srv
|
||||
COPY src/ /srv/src/
|
||||
COPY migrations/ /srv/migrations/
|
||||
# ревизия образа (меняется при rolling update; видна в журнале запуска)
|
||||
ARG REV=dev
|
||||
LABEL org.opencontainers.image.revision=$REV
|
||||
ENV APP_REVISION=$REV
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request,os,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2).status == 200 else 1)"]
|
||||
# exec-форма: сервер = PID 1. Состояние хранится в PostgreSQL/Valkey, процесс stateless;
|
||||
# масштабирование — репликами контейнера. SIGTERM: drain (readiness 503) -> graceful shutdown (см. app/serve.py).
|
||||
CMD ["python", "-m", "app.serve"]
|
||||
@@ -28,3 +28,18 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# --- собственные цели студента ---
|
||||
.PHONY: secrets up down student-test
|
||||
secrets:
|
||||
sh scripts/init-secrets.sh
|
||||
|
||||
up: secrets
|
||||
docker compose --env-file .env up -d --build
|
||||
|
||||
down:
|
||||
docker compose --env-file .env down
|
||||
|
||||
student-test:
|
||||
$(PYTHON) -m unittest discover -s student-tests -p "test_api.py" -v
|
||||
$(PYTHON) -m unittest discover -s student-tests -p "test_attachments.py" -v
|
||||
|
||||
@@ -1,37 +1,150 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Кушнаренко Николай Вячеславович, БПИ2302
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 2 «Согласованность двух хранилищ»
|
||||
- Commit/PR: проверенное решение `0150b91`, ветка `solution/bpi2302-kushnarenko` в [форке kolya](https://gitea.privcloud.tech/kolya/mtusi2026lab4); [PR #101](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/101) → `mtusi2026/mtusi2026lab4:main`
|
||||
- Использованные источники, библиотеки и помощь ИИ: Python 3.12, uvicorn, asyncpg, aiobotocore (S3), Pillow (проверка изображений), argon2-cffi; SeaweedFS 4.48 (S3); HAProxy; PostgreSQL; Valkey. Код lab3 перенесён. ИИ-ассистенты использовались для помощи с кодом, тестами и отчётом; Codex проверил подготовку к сдаче, усилил проверки исследования и выполнил реальные запуски 11.11.2026
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker Compose v2, bash (Git Bash на Windows), Python ≥ 3.9; на Windows `PYTHONUTF8=1` для `tests/*.py`.
|
||||
Версии: `python:3.12.12-slim-bookworm@sha256:593bd06e…`, `postgres:17.11-alpine@sha256:b0f9560a…`, `valkey/valkey:8.1.10-alpine@sha256:081c2f5c…`, `haproxy:3.0-alpine@sha256:56b887da…`, `chrislusf/seaweedfs:4.48@sha256:4e61d15f…`, зависимости — `requirements.txt` (закреплены, включая `aiobotocore==3.9.2`, `pillow==12.3.0`).
|
||||
|
||||
```sh
|
||||
sh scripts/init-secrets.sh # secrets/*: пароль БД, ключи S3 (admin и отдельные ключи API), seaweed_s3.json
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build # postgres, valkey, seaweedfs -> migrate + s3-init (один раз) -> api x2 -> lb
|
||||
curl http://127.0.0.1:8080/health/ready
|
||||
python3 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
sh scripts/restore-replicas.sh
|
||||
# собственные тесты (стенд должен быть запущен)
|
||||
python3 -m unittest discover -s student-tests -p "test_api.py" -v
|
||||
python3 -m unittest discover -s student-tests -p "test_attachments.py" -v
|
||||
python3 student-tests/test_idempotency.py
|
||||
python3 student-tests/test_topology.py
|
||||
python3 student-tests/test_s3_failures.py
|
||||
python3 student-tests/test_expiry.py
|
||||
python3 student-tests/test_dependencies.py
|
||||
```
|
||||
|
||||
Windows: `scripts/restart.cmd` и `scripts/stop-one.cmd` вместо `.sh`. Остановка с сохранением данных — `docker compose --env-file .env stop`; **намеренное удаление учебных данных** — `docker compose --env-file .env down -v` (volumes `pgdata`, `valkeydata`, `seaweeddata`).
|
||||
|
||||
**Выбор S3-сервера.** Используется **SeaweedFS**, разрешённый заданием. Образ закреплён тегом и digest. SeaweedFS запускается одним процессом (`weed server -filer -s3`), права задаются файлом identities (`secrets/seaweed_s3.json`).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
Клиент ─▶ HAProxy ─▶ API ×2 ─▶ PostgreSQL (метаданные вложений, сообщения, сессии, ключи идемпотентности)
|
||||
├─▶ SeaweedFS S3 (приватный bucket, байты вложений) [volume seaweeddata]
|
||||
└─▶ Valkey (кэш сессий)
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
| Роль | Сервис | Health | Volume | Сеть | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| S3 | `seaweedfs` (master+volume+filer+S3) | `wget …:9333/cluster/status` и `nc -z :8333` | `seaweeddata` | только `backend`, порты не опубликованы, **консоли/админки наружу нет** | `seaweed_s3.json` (secret) |
|
||||
| bootstrap | `s3-init` (one-shot) | код выхода 0 | — | `backend` | admin-ключи (только он), ключи API для проверки |
|
||||
| остальные роли | как в lab3 | | | | |
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
- **Данные.** Метаданные — таблицы `attachments` (id, chat_id, owner_id, display filename, content_type, size, sha256, status, error_code, s3_key, …) и `message_attachments (message_id, attachment_id, position)`; байты — в S3 под ключом **`att/<uuid>/original`, который назначает сервер**; `filename` используется только как отображаемое имя и нигде не входит в путь/ключ. Байты не хранятся ни в БД, ни в локальном каталоге API (временный буфер загрузки — `SpooledTemporaryFile`, удаляется после запроса).
|
||||
- **Загрузка** (`POST /chats/{id}/attachments?filename=…`, сырые bytes): авторизация → членство (чужой чат 404) → `filename` (1–255 code points; `/`, `\`, NUL, управляющие символы → 400) → `Content-Type` PNG/JPEG (иначе 415) → **размер ≤ 10 MiB проверяется по `Content-Length` до чтения и по счётчику при потоковом приёме** (413, соединение закрывается) → тело принимается порциями (до 1 MiB в RAM, далее временный файл) с подсчётом SHA-256 → проверка изображения **до полной распаковки**: сигнатура PNG/JPEG должна совпасть с заявленным типом (подмена MIME → 415), формат и размеры по заголовку Pillow, **>40 млн пикселей → 413** (decompression bomb отклоняется по заголовку), целостность: PNG — CRC всех чанков (`verify`) + потоковая распаковка IDAT с проверкой конца потока и ожидаемого размера данных, JPEG — декодирование в режиме draft (≈1/64 памяти) → запись в S3 → строка в PostgreSQL (`status=ready`, `201`). Ограничение одновременных загрузок на реплику (`UPLOAD_CONCURRENCY=4`, очередь 16, далее `429`).
|
||||
- **Скачивание** (`GET /attachments/{id}/content`, `?variant=original|thumbnail|crop|optimized`): авторизация → доступ по членству в чате вложения (постороннему/несуществующему — 404 без различий) → потоковая отдача из S3 порциями по 64 КиБ с истинным `Content-Type`, `Content-Length`, `X-Content-Type-Options: nosniff`, **без redirect**; недоступный S3 → `503`; несуществующий вариант → 404; невалидный `variant` → 400.
|
||||
- **Сообщения**: `attachment_ids` (0–10 уникальных UUID, по умолчанию `[]`; пустой текст допустим только с вложением). Вложение другого чата/неизвестное/чужое → 404; дубликаты, >10 → 400; идемпотентный повтор учитывает `attachment_ids` (нормализованное тело: порядок ключей JSON и отсутствие поля == `[]`). Проверка вложений и вставка — внутри той же серверной функции `send_message` (миграция 0004), то есть атомарно.
|
||||
- **Readiness API** = PostgreSQL. S3 в readiness **не включён осознанно**: при падении S3 текстовые сообщения продолжают работать (проверено тестом), вложения отвечают `503 + Retry-After`.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Согласованность S3 и PostgreSQL (механизм компенсации и сверки — уровень 4)
|
||||
|
||||
## Эксперименты и отказы
|
||||
Порядок записи: сначала объект в S3, затем строка в PostgreSQL. Окна отказа:
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
| Сбой | Результат | Исправление |
|
||||
| --- | --- | --- |
|
||||
| Клиент оборвал загрузку | ни объекта, ни строки (проверено тестом) | — |
|
||||
| S3 недоступен | 503, строка не создаётся | — |
|
||||
| Сбой БД после записи в S3 | клиент получает ошибку; объект **удаляется компенсацией** сразу (проверено внедрением сбоя `FAULT_INJECTION=db_after_put`) | компенсация |
|
||||
| Сбой БД и неудача удаления (или крах процесса между S3 и COMMIT) | «осиротевший» объект без строки | **reconciler** `python -m app.reconcile` |
|
||||
|
||||
**Reconciler** (`src/app/reconcile.py`): по умолчанию **dry-run**; сравнивает `attachments.s3_key` и листинг bucket. Находит: *orphan_object* (объект без строки; удаляется только если старше `grace` — по умолчанию 900 с, поэтому свежая, ещё не зафиксированная загрузка не затрагивается), *missing_object* (строка `ready` без объекта → `failed/object_missing`, **строка не удаляется**, сообщения сохраняются, скачивание даёт 409), *size_mismatch* (размер объекта ≠ `size_bytes` → `failed/size_mismatch`). Объект, у которого есть строка в БД, не удаляется никогда; перед удалением выполняется повторная проверка строки. Повторный запуск идемпотентен.
|
||||
|
||||
### Исправление отказа общей БД при подготовке к сдаче
|
||||
|
||||
В первом повторе 11.11.2026 тест остановки PostgreSQL обнаружил ответ вне JSON-контракта: после отрицательного readiness всех реплик HAProxy отдавал собственную ошибку вместо ответа приложения. Добавлен резервный backend по liveness, используемый только при отсутствии ready-реплик. Теперь живое приложение возвращает 503 с Retry-After и X-Request-Id, сохраняет 401 для запроса без токена и 200 для /health/live. Основная маршрутизация по readiness сохранена. Тест ждёт 5 с после остановки БД, чтобы гарантированно покрыть состояние после healthcheck балансировщика.
|
||||
|
||||
Также автоматические повторы HAProxy ограничены ошибкой установления соединения: ответ/таймаут уже отправленного upload POST не повторяется, поскольку контракт не делает загрузку идемпотентной. Описание директив: [HAProxy 3.0 Configuration Manual](https://docs.haproxy.org/3.0/configuration.html). Первичный сбой сохранён в `evidence/checks_before_fix_2026-11-11.txt`; полный итоговый прогон — в `evidence/checks_2026-11-11.txt`.
|
||||
|
||||
## Доказательства критериев — повторная проверка 11.11.2026
|
||||
|
||||
Проверены текущая ветка решения и исходный skeleton `ec540a1`. README задания, COURSE, CONTRIBUTING, публичные `tests/` и `contracts/` сохранены без изменений. Команды и фактический вывод: [evidence/checks_2026-11-11.txt](evidence/checks_2026-11-11.txt). В Windows выполнены Python-команды, соответствующие `make check`, `make validate` и `make test`; GNU Make для проверки не требуется.
|
||||
|
||||
| Критерий | Проверка | Результат |
|
||||
| --- | --- | --- |
|
||||
| Контракт и публичный сценарий | check_contract, validate_openapi, smoke | PASS |
|
||||
| API, авторизация и идемпотентность между репликами | test_api + test_idempotency | 12 + 9 тестов OK |
|
||||
| Изображения, лимиты, MIME, filename, ACL, чужой чат, прерывание upload, private bucket | test_attachments | 11 тестов OK; пользовательские ошибки не дают 500; анонимный S3 GET → 403 |
|
||||
| Реплики, порты, лимиты, непривилегированный пользователь | test_topology | 6 тестов OK |
|
||||
| Компенсация, dry-run/apply, свежие orphan, используемые вложения, отказ S3 | test_s3_failures | 4 теста OK; защищённый файл связан с сообщением, ссылка сохранена |
|
||||
| Истечение сессии, недоступность PostgreSQL и Valkey | test_expiry + test_dependencies | 2 + 2 теста OK |
|
||||
| Сохранность при restart и остановке обслужившей реплики | tests/resilience.py | persistence/failover PASS; восстановление записи 2.329 с, 105 успешных записей, 0 неуспешных попыток |
|
||||
| Идемпотентный bootstrap, отдельные ограниченные ключи API | два повторных запуска s3-init | существующий bucket сохранён; anonymous → 403; create_bucket ключом API → AccessDenied |
|
||||
| ≥10 параллельных загрузок и пик памяти | research/upload_memory.py | 12 клиентов; все файлы приняты и SHA-256 совпали; OOM нет |
|
||||
| Трек 2: обнаружение и исправление расхождений, стоимость обхода | research/reconcile_bench.py | 20 orphan / 20 missing / 5 size mismatch; безопасное исправление и повтор без изменений |
|
||||
|
||||
Всего **46 собственных тестов**. После экспериментов дополнительно повторены persistence и smoke. В [lifecycle.json](evidence/recheck_2026-11-11/lifecycle.json) зафиксировано изменение StartedAt всех сервисов (включая обе API-реплики и S3), сохранение имён volumes и healthy после restart.
|
||||
|
||||
## Эксперименты и отказы (трек 2)
|
||||
|
||||
**Гипотеза.** Reconciler находит внедрённые расхождения в обе стороны; dry-run ничего не меняет; apply удаляет только старые объекты без строки в БД, помечает потерянные/повреждённые вложения failed и сохраняет свежие объекты и файлы сообщений. Повтор apply не вносит изменений. Стоимость полного обхода увеличивается с числом объектов.
|
||||
|
||||
**Методика и окружение.** Windows, Docker Desktop с Linux-контейнерами, Python клиента 3.11.9, aiohttp 3.14.3, Pillow 12.3.0; приложение — Python 3.12.12. Образы и лимиты закреплены в compose.yaml. Во время исследования на машине оставался другой Docker-проект; абсолютные времена зависят от фоновой нагрузки. Один измерительный прогон на точку, статистическая значимость различий не оценивалась. Команды воспроизведения — [student-tests/README.md](student-tests/README.md).
|
||||
|
||||
В клиенте исследования сверки отключено повторное использование HTTP-соединений: блокирующий вызов Docker мог длиться дольше keep-alive сервера, и первая попытка остановилась с ServerDisconnectedError до внедрения расхождений. После исправления выполнен полный повтор сверки; исходный вывод сохранён в `evidence/recheck_2026-11-11/commands_before_client_fix.txt`. Замеры памяти не повторялись: они завершились успешно и не зависели от этого клиента. SHA-256 проверенных исходников записаны в `source_metadata.json` рядом с результатами.
|
||||
|
||||
### Память при одновременных загрузках
|
||||
|
||||
12 клиентов загружают по 8.73 MiB PNG (шум 1700×1700). Для каждой точки создаётся одна новая API-реплика, 1 CPU / 256 MiB; меняется только UPLOAD_CONCURRENCY. Пик — VmHWM процесса API, это не суммарная память всех контейнеров. После измерения каждый принятый файл скачивается и проверяется SHA-256. В конце восстановлены две реплики с UPLOAD_CONCURRENCY=4.
|
||||
|
||||
| UPLOAD_CONCURRENCY | Пик RSS, MiB | Загрузка всех файлов, с | Принято и цело | OOMKilled |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | 99.2 | 1.31 | 12/12 | false |
|
||||
| 4 | 119.0 | 1.18 | 12/12 | false |
|
||||
| 12 | 121.2 | 1.14 | 12/12 | false |
|
||||
|
||||
### Сверка двух хранилищ
|
||||
|
||||
К существующим синтетическим данным добавлены 3000 вложений тремя порциями по 1000. После каждой порции измерен dry-run. Затем созданы 20 orphan в S3, удалены объекты 20 существующих вложений и изменён размер ещё 5. **200 других вложений прикреплены к 20 сообщениям** (по 10); перед исправлением сохранены идентификаторы сообщений и их attachment_ids.
|
||||
|
||||
После паузы больше 5 с выполняются два dry-run, затем apply с grace=5 параллельно с задачей загрузки 200 новых файлов, затем повторный apply. Рабочее значение grace — 900 с; короткий grace используется только в эксперименте. Отдельный тест детерминированно оставляет свежий orphan и подтверждает, что `--apply --grace 900` его не удаляет.
|
||||
|
||||
| Проверка | Фактический результат |
|
||||
| --- | --- |
|
||||
| dry-run ×2 | ровно 20 orphan, 20 missing, 5 size mismatch; fixed=0/0; повторяемо |
|
||||
| apply | удалено 20 старых orphan, 25 строк помечены failed (20 object_missing, 5 size_mismatch) |
|
||||
| Новые загрузки | 200/200 приняты и скачаны с совпадающей SHA-256 |
|
||||
| Вложения сообщений | 200/200 целы; все 20 сообщений и attachment_ids неизменны |
|
||||
| Повтор apply | 0 удалений, 0 изменений статусов, новых расхождений нет |
|
||||
|
||||
| Строк БД | Объектов S3 | Запрос БД, мс | Листинг S3, мс | Весь обход, мс |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 19554 | 19530 | 20.2 | 3540.6 | 3649.4 |
|
||||
| 20554 | 20530 | 19.0 | 3697.2 | 3806.4 |
|
||||
| 21554 | 21530 | 21.2 | 3939.1 | 4055.0 |
|
||||
|
||||
На проверенном наборе основную стоимость даёт листинг S3; результаты трёх точек не доказывают линейное время на произвольном объёме. Алгоритм хранит снимки метаданных и листинга в памяти: время и дополнительная память — O(N). Сырой результат: [upload_memory.json](evidence/recheck_2026-11-11/upload_memory.json), [track2_reconcile.json](evidence/recheck_2026-11-11/track2_reconcile.json); вывод команд — [commands.txt](evidence/recheck_2026-11-11/commands.txt).
|
||||
|
||||
**Границы.** Сверяются наличие и размер, но не содержимое S3-объектов: подмена байтов без изменения размера этим reconciler не обнаруживается. Целостность контрольных файлов подтверждает скачивание с SHA-256. Потерянные байты не восстанавливаются, строка остаётся failed. Grace должен превышать время загрузки и таймаутов; для рабочих запусков принят 900 с. S3 и PostgreSQL — по одному узлу. Старые октябрьские результаты сохранены в `evidence/checks.txt`, `evidence/upload_memory.json` и `evidence/track2_reconcile.json`; текущие выводы опираются на повтор 11.11.2026.
|
||||
|
||||
### Отказы
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановка S3 | загрузка/скачивание 503 + `Retry-After`, текст работает, ready=200 | OK (`test_s3_down…`) | после старта S3 — сразу; метаданные не накапливают «битых» строк |
|
||||
| Остановка одной реплики после upload | файл скачивается другой репликой | failover PASS (восстановление записи 2.329 с) | потерь нет |
|
||||
| Рестарт всего стенда | метаданные и оригинал на месте | persistence PASS | < 90 с |
|
||||
| Сбой БД после записи в S3 | объект не остаётся | удаляется компенсацией; при сбое компенсации — reconciler | потерь подтверждённых данных нет |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один узел S3 и один PostgreSQL (SPOF); репликации нет. Загрузка проксируется через API (прямые подписанные URL не реализованы — трек 1 не выбран).
|
||||
- Проверка содержимого JPEG — декодирование в режиме draft (1/8), PNG — потоковая проверка; полная валидация пикселей не выполняется (это не нужно для отдачи оригинала).
|
||||
- Подтверждено экспериментом: обнаружение/исправление расхождений, защита свежих и используемых вложений, стоимость обхода на трёх размерах набора, ограничение памяти при 12 одновременных загрузках. Гипотеза без доказательства: поведение reconciler на бакетах > 100 тыс. объектов.
|
||||
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
# Стенд lab4: HAProxy -> API x N -> PostgreSQL (метаданные) + SeaweedFS (приватный S3, байты) + Valkey (кэш сессий).
|
||||
# Секреты: scripts/init-secrets.sh создаёт secrets/* (в Git не попадают). Запуск и проверки — см. REPORT.md.
|
||||
name: mtusi-lab4
|
||||
|
||||
x-common: &common
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options: {max-size: "10m", max-file: "3"}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
<<: *common
|
||||
image: postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=128MB", "-c", "log_min_duration_statement=500"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
mem_limit: 512m
|
||||
cpus: ${POSTGRES_CPUS:-1.0}
|
||||
networks: [backend]
|
||||
|
||||
valkey:
|
||||
<<: *common
|
||||
image: valkey/valkey:8.1.10-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec",
|
||||
"--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl", "--save", ""]
|
||||
volumes:
|
||||
- valkeydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
networks: [backend]
|
||||
|
||||
# Приватное S3-совместимое хранилище SeaweedFS (all-in-one: master + volume + filer + S3-шлюз).
|
||||
# Порты наружу не публикуются: доступ только из сети backend и только по ключам из s3.json (identities).
|
||||
# Анонимный доступ запрещён (при заданных identities запрос без подписи получает 403).
|
||||
seaweedfs:
|
||||
<<: *common
|
||||
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command: ["exec weed server -dir=/data -ip=seaweedfs -master.volumeSizeLimitMB=512 -volume.max=30 -filer -s3 -s3.port=8333 -s3.config=/run/secrets/seaweed_s3_config"]
|
||||
secrets: [seaweed_s3_config]
|
||||
volumes:
|
||||
- seaweeddata:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status && nc -z 127.0.0.1 8333"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 10s
|
||||
mem_limit: 768m
|
||||
cpus: ${S3_CPUS:-1.0}
|
||||
networks: [backend]
|
||||
|
||||
# Идемпотентный bootstrap: bucket создаётся, если его нет; проверяется, что анонимный доступ запрещён и что ключ API работает.
|
||||
s3-init:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
command: ["python", "-m", "app.s3_bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://seaweedfs:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key
|
||||
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
secrets: [s3_admin_access_key, s3_admin_secret_key, s3_access_key, s3_secret_key]
|
||||
depends_on:
|
||||
seaweedfs: {condition: service_healthy}
|
||||
restart: "no"
|
||||
networks: [backend]
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
restart: "no"
|
||||
networks: [backend]
|
||||
|
||||
api:
|
||||
<<: *common
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
environment:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
VALKEY_URL: redis://valkey:6379/0
|
||||
S3_ENDPOINT: http://seaweedfs:8333
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-4}
|
||||
MAX_INFLIGHT: ${MAX_INFLIGHT:-64}
|
||||
ACCESS_LOG: ${ACCESS_LOG:-1}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
|
||||
FAULT_INJECTION: ${FAULT_INJECTION:-}
|
||||
secrets: [database_url, s3_access_key, s3_secret_key]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
valkey: {condition: service_healthy}
|
||||
stop_grace_period: 15s
|
||||
mem_limit: ${API_MEM:-256m}
|
||||
cpus: ${API_CPUS:-1.0}
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
networks: [backend]
|
||||
|
||||
lb:
|
||||
<<: *common
|
||||
image: haproxy:3.0-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
- api
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/healthz"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 6
|
||||
mem_limit: 128m
|
||||
cpus: ${LB_CPUS:-1.0}
|
||||
stop_grace_period: 10s
|
||||
networks: [backend]
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
seaweed_s3_config:
|
||||
file: ./secrets/seaweed_s3.json
|
||||
s3_admin_access_key:
|
||||
file: ./secrets/s3_admin_access_key
|
||||
s3_admin_secret_key:
|
||||
file: ./secrets/s3_admin_secret_key
|
||||
s3_access_key:
|
||||
file: ./secrets/s3_access_key
|
||||
s3_secret_key:
|
||||
file: ./secrets/s3_secret_key
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
seaweeddata:
|
||||
|
||||
networks:
|
||||
backend:
|
||||
@@ -0,0 +1,48 @@
|
||||
## Лабораторная 4
|
||||
|
||||
Автор / группа: Кушнаренко Николай Вячеславович, БПИ2302
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: Трек 2 — согласованность двух хранилищ
|
||||
|
||||
Что реализовано и что перенесено:
|
||||
Из ЛР3 перенесены PostgreSQL, сессии, Valkey, HAProxy, две API-реплики, идемпотентность сообщений, миграции и сценарии отказов. Добавлены приватное S3-хранилище SeaweedFS, метаданные вложений в PostgreSQL, binary upload PNG/JPEG и авторизованный download через API с побайтной сохранностью. Ограничения: 10 MiB, 40 млн пикселей, проверка MIME по содержимому и безопасного filename, потоковая обработка и лимит параллельных загрузок. Реализованы attachment_ids в сообщениях, компенсация ошибки между S3/БД и reconciler с dry-run, grace period и повторным безопасным запуском. Bootstrap идемпотентен, API использует отдельные ключи с ограниченными правами. Исправлена маршрутизация HAProxy при отказе общей БД: даже после снятия ready всех реплик живые API сохраняют контракт ошибок и liveness; повтор отправленного upload POST отключён.
|
||||
|
||||
Команды запуска из чистого clone (bash / Git Bash):
|
||||
|
||||
```sh
|
||||
sh scripts/init-secrets.sh
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
python3 scripts/check_contract.py
|
||||
python3 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
sh scripts/restore-replicas.sh
|
||||
```
|
||||
На Windows: PYTHONUTF8=1 и hooks scripts/restart.cmd, scripts/stop-one.cmd. Полная инструкция, собственные тесты и эксперименты — в [REPORT.md](https://gitea.privcloud.tech/kolya/mtusi2026lab4/src/branch/solution/bpi2302-kushnarenko/REPORT.md) и [student-tests/README.md](https://gitea.privcloud.tech/kolya/mtusi2026lab4/src/branch/solution/bpi2302-kushnarenko/student-tests/README.md).
|
||||
|
||||
Результаты проверок (повтор 11.11.2026):
|
||||
|
||||
- check_contract, полная валидация OpenAPI и публичный smoke — PASS;
|
||||
- 46 собственных тестов — OK: авторизация, идемпотентность, вложения/ACL, топология, отказы S3/БД/кэша, TTL сессий;
|
||||
- persistence и failover — PASS; восстановление записи 2.329 с, 105 успешных записей, 0 неуспешных попыток; сохранность оригинала проверена по SHA-256;
|
||||
- bootstrap дважды — OK: анонимный доступ 403, создание bucket ключом API запрещено;
|
||||
- 12 параллельных загрузок по 8.73 MiB: пик RSS 99.2–121.2 MiB при лимите 256 MiB; все файлы целы, OOM нет;
|
||||
- трек 2: обнаружены ровно 20 orphan / 20 missing / 5 size mismatch; dry-run ничего не меняет; apply удаляет 20 старых orphan и помечает 25 строк failed; повтор без изменений;
|
||||
- свежий orphan защищён grace period; 200 новых файлов и 200 вложений в 20 сообщениях сохранены, attachment_ids неизменны. Стоимость обхода: 19530 объектов → 3.65 с; 20530 объектов → 3.81 с; 21530 объектов → 4.05 с.
|
||||
|
||||
Доказательства: [checks_2026-11-11.txt](https://gitea.privcloud.tech/kolya/mtusi2026lab4/src/branch/solution/bpi2302-kushnarenko/evidence/checks_2026-11-11.txt), [JSON и команды исследований](https://gitea.privcloud.tech/kolya/mtusi2026lab4/src/branch/solution/bpi2302-kushnarenko/evidence/recheck_2026-11-11), отчёт [REPORT.md](https://gitea.privcloud.tech/kolya/mtusi2026lab4/src/branch/solution/bpi2302-kushnarenko/REPORT.md) и [Word](https://gitea.privcloud.tech/kolya/mtusi2026lab4/src/branch/solution/bpi2302-kushnarenko/docs/report_lab4.docx).
|
||||
|
||||
Ограничения / известные ошибки:
|
||||
S3 и PostgreSQL работают по одному узлу; потерянные байты reconciler не восстанавливает. Сверяется наличие и размер, подмена содержимого при том же размере не обнаруживается. Полный обход требует O(N) времени и памяти. Замеры сделаны на одном компьютере, по одному прогону на точку, с фоновой нагрузкой другого Docker-проекта. Прямой presigned upload не реализован — выбран трек 2.
|
||||
|
||||
Источники и использование ИИ:
|
||||
Python, uvicorn, asyncpg, aiobotocore, Pillow, argon2-cffi, SeaweedFS, PostgreSQL, Valkey, HAProxy; первичные материалы — в README задания. ИИ-ассистенты использовались для помощи с кодом, тестами и отчётом; Codex выполнил проверку подготовки к сдаче, усилил доказательства исследования и проверил результаты реальными запусками.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены через эквивалентные Python-команды; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Binary file not shown.
@@ -0,0 +1,46 @@
|
||||
$ python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
$ python3 tests/smoke.py
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
# контейнеры до перезапуска:
|
||||
/mtusi-lab4-api-3 StartedAt=2026-10-02T23:34:55.153397385Z
|
||||
/mtusi-lab4-api-4 StartedAt=2026-10-02T23:34:55.348327032Z
|
||||
/mtusi-lab4-seaweedfs-1 StartedAt=2026-10-02T23:32:25.854895161Z
|
||||
/mtusi-lab4-postgres-1 StartedAt=2026-10-02T23:32:25.588611658Z
|
||||
/mtusi-lab2-postgres-1 StartedAt=2026-10-02T22:32:35.614573178Z
|
||||
/mtusi-lab4-valkey-1 StartedAt=2026-10-02T23:32:25.398680344Z
|
||||
/mtusi-lab2-valkey-1 StartedAt=2026-10-02T22:32:35.599034066Z
|
||||
$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh (Windows: restart.cmd; проверяются и метаданные, и исходный файл в S3)
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
# контейнеры после перезапуска:
|
||||
/mtusi-lab4-api-3 StartedAt=2026-10-02T23:39:43.36427692Z
|
||||
/mtusi-lab4-api-4 StartedAt=2026-10-02T23:39:43.220921592Z
|
||||
/mtusi-lab4-seaweedfs-1 StartedAt=2026-10-02T23:39:40.130628797Z
|
||||
/mtusi-lab4-postgres-1 StartedAt=2026-10-02T23:39:39.628503986Z
|
||||
/mtusi-lab2-postgres-1 StartedAt=2026-10-02T22:32:35.614573178Z
|
||||
/mtusi-lab4-valkey-1 StartedAt=2026-10-02T23:39:39.858223769Z
|
||||
/mtusi-lab2-valkey-1 StartedAt=2026-10-02T22:32:35.599034066Z
|
||||
$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh (1-я реплика; файл скачивается другой репликой)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ ... (2-я реплика)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 107, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ python -m unittest test_api.py
|
||||
Ran 12 tests in 2.472s
|
||||
OK
|
||||
$ python -m unittest test_idempotency.py
|
||||
Ran 9 tests in 1.916s
|
||||
OK
|
||||
$ python -m unittest test_attachments.py
|
||||
Ran 11 tests in 4.632s
|
||||
OK
|
||||
$ python student-tests/test_topology.py
|
||||
Ran 6 tests in 1.485s
|
||||
OK
|
||||
$ python student-tests/test_s3_failures.py (остановка S3, сбои БД после записи в S3, reconcile)
|
||||
Ran 4 tests in 156.651s
|
||||
OK
|
||||
@@ -0,0 +1,167 @@
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
|
||||
$ python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_api.py -v
|
||||
test_bearer_rules (test_api.AuthTest.test_bearer_rules) ... ok
|
||||
test_concurrent_registration_of_same_username (test_api.AuthTest.test_concurrent_registration_of_same_username) ... ok
|
||||
test_logout_revokes_in_database_and_cache (test_api.AuthTest.test_logout_revokes_in_database_and_cache) ... ok
|
||||
test_password_is_not_exposed_or_stored_plain (test_api.AuthTest.test_password_is_not_exposed_or_stored_plain) ... ok
|
||||
test_register_validation (test_api.AuthTest.test_register_validation) ... ok
|
||||
test_session_shape_entropy_and_logout (test_api.AuthTest.test_session_shape_entropy_and_logout) ... ok
|
||||
test_token_only_stored_as_hash (test_api.AuthTest.test_token_only_stored_as_hash) ... ok
|
||||
test_wrong_password_and_unknown_user_are_indistinguishable (test_api.AuthTest.test_wrong_password_and_unknown_user_are_indistinguishable) ... ok
|
||||
test_chat_creation_is_atomic (test_api.DataTest.test_chat_creation_is_atomic) ... ok
|
||||
test_foreign_chat_empty_message_bad_json (test_api.DataTest.test_foreign_chat_empty_message_bad_json) ... ok
|
||||
test_multi_page_history_and_cursor_binding (test_api.DataTest.test_multi_page_history_and_cursor_binding) ... ok
|
||||
test_parallel_writes_from_many_clients (test_api.DataTest.test_parallel_writes_from_many_clients) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 12 tests in 2.711s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_idempotency.py -v
|
||||
test_concurrent_same_key_creates_exactly_one_message_across_replicas (test_idempotency.IdempotencyTest.test_concurrent_same_key_creates_exactly_one_message_across_replicas) ... ok
|
||||
test_different_body_same_key_is_409 (test_idempotency.IdempotencyTest.test_different_body_same_key_is_409) ... ok
|
||||
test_expired_key_is_reusable_after_ttl (test_idempotency.IdempotencyTest.test_expired_key_is_reusable_after_ttl) ... ok
|
||||
test_key_validation (test_idempotency.IdempotencyTest.test_key_validation) ... ok
|
||||
test_replay_rechecks_session_and_membership (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_membership) ... ok
|
||||
test_replay_returns_same_message_and_creates_no_duplicate (test_idempotency.IdempotencyTest.test_replay_returns_same_message_and_creates_no_duplicate) ... ok
|
||||
test_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_scope_is_user_and_chat) ... ok
|
||||
test_semantically_equal_body_is_replayed (test_idempotency.IdempotencyTest.test_semantically_equal_body_is_replayed) ... ok
|
||||
test_session_is_shared_between_replicas_without_relogin (test_idempotency.IdempotencyTest.test_session_is_shared_between_replicas_without_relogin) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 9 tests in 1.920s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_attachments.py -v
|
||||
test_aborted_upload_leaves_nothing (test_attachments.AttachmentTest.test_aborted_upload_leaves_nothing) ... ok
|
||||
test_bucket_is_private_outside_the_api (test_attachments.AttachmentTest.test_bucket_is_private_outside_the_api) ... ok
|
||||
test_corrupted_and_spoofed_files_are_415_not_500 (test_attachments.AttachmentTest.test_corrupted_and_spoofed_files_are_415_not_500) ... ok
|
||||
test_dangerous_filenames_are_400 (test_attachments.AttachmentTest.test_dangerous_filenames_are_400) ... ok
|
||||
test_decompression_bomb_header_is_413 (test_attachments.AttachmentTest.test_decompression_bomb_header_is_413) ... ok
|
||||
test_empty_body_is_400 (test_attachments.AttachmentTest.test_empty_body_is_400) ... ok
|
||||
test_foreign_user_cannot_upload_read_or_download (test_attachments.AttachmentTest.test_foreign_user_cannot_upload_read_or_download) ... ok
|
||||
test_idempotent_replay_with_attachments (test_attachments.AttachmentTest.test_idempotent_replay_with_attachments) ... ok
|
||||
test_message_with_attachments_and_chat_binding (test_attachments.AttachmentTest.test_message_with_attachments_and_chat_binding) ... ok
|
||||
test_png_and_jpeg_roundtrip_with_checksum (test_attachments.AttachmentTest.test_png_and_jpeg_roundtrip_with_checksum) ... ok
|
||||
test_too_large_is_413_and_nothing_is_stored (test_attachments.AttachmentTest.test_too_large_is_413_and_nothing_is_stored) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 11 tests in 4.285s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_topology.py -v
|
||||
test_api_runs_as_non_root (test_topology.TopologyTest.test_api_runs_as_non_root) ... ok
|
||||
test_at_least_two_api_replicas_without_published_ports (test_topology.TopologyTest.test_at_least_two_api_replicas_without_published_ports) ... ok
|
||||
test_no_fixed_container_names (test_topology.TopologyTest.test_no_fixed_container_names) ... ok
|
||||
test_only_lb_is_published_and_only_on_localhost (test_topology.TopologyTest.test_only_lb_is_published_and_only_on_localhost) ... ok
|
||||
test_resource_limits_are_set (test_topology.TopologyTest.test_resource_limits_are_set) ... ok
|
||||
test_stop_grace_period_covers_drain (test_topology.TopologyTest.test_stop_grace_period_covers_drain) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 6 tests in 1.649s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_s3_failures.py -v
|
||||
test_db_failure_after_s3_write_is_compensated (test_s3_failures.S3FailureTest.test_db_failure_after_s3_write_is_compensated) ... ok
|
||||
test_missing_object_is_reported_and_attachment_marked_failed (test_s3_failures.S3FailureTest.test_missing_object_is_reported_and_attachment_marked_failed) ... ok
|
||||
test_orphan_left_by_failed_compensation_is_found_and_cleaned_safely (test_s3_failures.S3FailureTest.test_orphan_left_by_failed_compensation_is_found_and_cleaned_safely) ... ok
|
||||
test_s3_down_gives_503_text_path_keeps_working (test_s3_failures.S3FailureTest.test_s3_down_gives_503_text_path_keeps_working) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 4 tests in 189.512s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 105, "failed_attempts": 0, "first_recovery_seconds": 2.329}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env start api
|
||||
Container mtusi-lab4-postgres-1 Waiting
|
||||
Container mtusi-lab4-seaweedfs-1 Waiting
|
||||
Container mtusi-lab4-seaweedfs-1 Healthy
|
||||
Container mtusi-lab4-s3-init-1 Starting
|
||||
Container mtusi-lab4-postgres-1 Healthy
|
||||
Container mtusi-lab4-migrate-1 Starting
|
||||
Container mtusi-lab4-migrate-1 Started
|
||||
Container mtusi-lab4-s3-init-1 Started
|
||||
Container mtusi-lab4-s3-init-1 Waiting
|
||||
Container mtusi-lab4-valkey-1 Waiting
|
||||
Container mtusi-lab4-migrate-1 Waiting
|
||||
Container mtusi-lab4-valkey-1 Healthy
|
||||
Container mtusi-lab4-migrate-1 Exited
|
||||
Container mtusi-lab4-s3-init-1 Exited
|
||||
Container mtusi-lab4-api-5 Starting
|
||||
Container mtusi-lab4-api-5 Started
|
||||
exit=0
|
||||
|
||||
$ python student-tests/test_expiry.py
|
||||
..
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 19.361s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python student-tests/test_dependencies.py
|
||||
..
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 19.594s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env run --rm -T s3-init
|
||||
Container mtusi-lab4-seaweedfs-1 Running
|
||||
Container mtusi-lab4-seaweedfs-1 Waiting
|
||||
Container mtusi-lab4-seaweedfs-1 Healthy
|
||||
Container mtusi-lab4-s3-init-run-da275e60a199 Creating
|
||||
Container mtusi-lab4-s3-init-run-da275e60a199 Created
|
||||
bucket messenger-attachments уже существует
|
||||
анонимный доступ запрещён (HTTP 403)
|
||||
ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied)
|
||||
bootstrap завершён
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env run --rm -T s3-init
|
||||
Container mtusi-lab4-seaweedfs-1 Running
|
||||
Container mtusi-lab4-seaweedfs-1 Waiting
|
||||
Container mtusi-lab4-seaweedfs-1 Healthy
|
||||
Container mtusi-lab4-s3-init-run-cc1f19e0054b Creating
|
||||
Container mtusi-lab4-s3-init-run-cc1f19e0054b Created
|
||||
bucket messenger-attachments уже существует
|
||||
анонимный доступ запрещён (HTTP 403)
|
||||
ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied)
|
||||
bootstrap завершён
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,159 @@
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
|
||||
$ python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_api.py -v
|
||||
test_bearer_rules (test_api.AuthTest.test_bearer_rules) ... ok
|
||||
test_concurrent_registration_of_same_username (test_api.AuthTest.test_concurrent_registration_of_same_username) ... ok
|
||||
test_logout_revokes_in_database_and_cache (test_api.AuthTest.test_logout_revokes_in_database_and_cache) ... ok
|
||||
test_password_is_not_exposed_or_stored_plain (test_api.AuthTest.test_password_is_not_exposed_or_stored_plain) ... ok
|
||||
test_register_validation (test_api.AuthTest.test_register_validation) ... ok
|
||||
test_session_shape_entropy_and_logout (test_api.AuthTest.test_session_shape_entropy_and_logout) ... ok
|
||||
test_token_only_stored_as_hash (test_api.AuthTest.test_token_only_stored_as_hash) ... ok
|
||||
test_wrong_password_and_unknown_user_are_indistinguishable (test_api.AuthTest.test_wrong_password_and_unknown_user_are_indistinguishable) ... ok
|
||||
test_chat_creation_is_atomic (test_api.DataTest.test_chat_creation_is_atomic) ... ok
|
||||
test_foreign_chat_empty_message_bad_json (test_api.DataTest.test_foreign_chat_empty_message_bad_json) ... ok
|
||||
test_multi_page_history_and_cursor_binding (test_api.DataTest.test_multi_page_history_and_cursor_binding) ... ok
|
||||
test_parallel_writes_from_many_clients (test_api.DataTest.test_parallel_writes_from_many_clients) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 12 tests in 2.524s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_idempotency.py -v
|
||||
test_concurrent_same_key_creates_exactly_one_message_across_replicas (test_idempotency.IdempotencyTest.test_concurrent_same_key_creates_exactly_one_message_across_replicas) ... ok
|
||||
test_different_body_same_key_is_409 (test_idempotency.IdempotencyTest.test_different_body_same_key_is_409) ... ok
|
||||
test_expired_key_is_reusable_after_ttl (test_idempotency.IdempotencyTest.test_expired_key_is_reusable_after_ttl) ... ok
|
||||
test_key_validation (test_idempotency.IdempotencyTest.test_key_validation) ... ok
|
||||
test_replay_rechecks_session_and_membership (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_membership) ... ok
|
||||
test_replay_returns_same_message_and_creates_no_duplicate (test_idempotency.IdempotencyTest.test_replay_returns_same_message_and_creates_no_duplicate) ... ok
|
||||
test_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_scope_is_user_and_chat) ... ok
|
||||
test_semantically_equal_body_is_replayed (test_idempotency.IdempotencyTest.test_semantically_equal_body_is_replayed) ... ok
|
||||
test_session_is_shared_between_replicas_without_relogin (test_idempotency.IdempotencyTest.test_session_is_shared_between_replicas_without_relogin) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 9 tests in 1.769s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_attachments.py -v
|
||||
test_aborted_upload_leaves_nothing (test_attachments.AttachmentTest.test_aborted_upload_leaves_nothing) ... ok
|
||||
test_bucket_is_private_outside_the_api (test_attachments.AttachmentTest.test_bucket_is_private_outside_the_api) ... ok
|
||||
test_corrupted_and_spoofed_files_are_415_not_500 (test_attachments.AttachmentTest.test_corrupted_and_spoofed_files_are_415_not_500) ... ok
|
||||
test_dangerous_filenames_are_400 (test_attachments.AttachmentTest.test_dangerous_filenames_are_400) ... ok
|
||||
test_decompression_bomb_header_is_413 (test_attachments.AttachmentTest.test_decompression_bomb_header_is_413) ... ok
|
||||
test_empty_body_is_400 (test_attachments.AttachmentTest.test_empty_body_is_400) ... ok
|
||||
test_foreign_user_cannot_upload_read_or_download (test_attachments.AttachmentTest.test_foreign_user_cannot_upload_read_or_download) ... ok
|
||||
test_idempotent_replay_with_attachments (test_attachments.AttachmentTest.test_idempotent_replay_with_attachments) ... ok
|
||||
test_message_with_attachments_and_chat_binding (test_attachments.AttachmentTest.test_message_with_attachments_and_chat_binding) ... ok
|
||||
test_png_and_jpeg_roundtrip_with_checksum (test_attachments.AttachmentTest.test_png_and_jpeg_roundtrip_with_checksum) ... ok
|
||||
test_too_large_is_413_and_nothing_is_stored (test_attachments.AttachmentTest.test_too_large_is_413_and_nothing_is_stored) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 11 tests in 4.352s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_topology.py -v
|
||||
test_api_runs_as_non_root (test_topology.TopologyTest.test_api_runs_as_non_root) ... ok
|
||||
test_at_least_two_api_replicas_without_published_ports (test_topology.TopologyTest.test_at_least_two_api_replicas_without_published_ports) ... ok
|
||||
test_no_fixed_container_names (test_topology.TopologyTest.test_no_fixed_container_names) ... ok
|
||||
test_only_lb_is_published_and_only_on_localhost (test_topology.TopologyTest.test_only_lb_is_published_and_only_on_localhost) ... ok
|
||||
test_resource_limits_are_set (test_topology.TopologyTest.test_resource_limits_are_set) ... ok
|
||||
test_stop_grace_period_covers_drain (test_topology.TopologyTest.test_stop_grace_period_covers_drain) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 6 tests in 1.580s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python -m unittest discover -s student-tests -p test_s3_failures.py -v
|
||||
test_db_failure_after_s3_write_is_compensated (test_s3_failures.S3FailureTest.test_db_failure_after_s3_write_is_compensated) ... ok
|
||||
test_missing_object_is_reported_and_attachment_marked_failed (test_s3_failures.S3FailureTest.test_missing_object_is_reported_and_attachment_marked_failed) ... ok
|
||||
test_orphan_left_by_failed_compensation_is_found_and_cleaned_safely (test_s3_failures.S3FailureTest.test_orphan_left_by_failed_compensation_is_found_and_cleaned_safely) ... ok
|
||||
test_s3_down_gives_503_text_path_keeps_working (test_s3_failures.S3FailureTest.test_s3_down_gives_503_text_path_keeps_working) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 4 tests in 193.833s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ docker compose --env-file .env start api
|
||||
Container mtusi-lab4-postgres-1 Waiting
|
||||
Container mtusi-lab4-seaweedfs-1 Waiting
|
||||
Container mtusi-lab4-seaweedfs-1 Healthy
|
||||
Container mtusi-lab4-postgres-1 Healthy
|
||||
Container mtusi-lab4-s3-init-1 Starting
|
||||
Container mtusi-lab4-migrate-1 Starting
|
||||
Container mtusi-lab4-migrate-1 Started
|
||||
Container mtusi-lab4-s3-init-1 Started
|
||||
Container mtusi-lab4-s3-init-1 Waiting
|
||||
Container mtusi-lab4-valkey-1 Waiting
|
||||
Container mtusi-lab4-migrate-1 Waiting
|
||||
Container mtusi-lab4-valkey-1 Healthy
|
||||
Container mtusi-lab4-migrate-1 Exited
|
||||
Container mtusi-lab4-s3-init-1 Exited
|
||||
Container mtusi-lab4-api-4 Starting
|
||||
Container mtusi-lab4-api-4 Started
|
||||
exit=0
|
||||
|
||||
$ python student-tests/test_expiry.py
|
||||
..
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 23.614s
|
||||
|
||||
OK
|
||||
exit=0
|
||||
|
||||
$ python student-tests/test_dependencies.py
|
||||
E.
|
||||
======================================================================
|
||||
ERROR: test_postgres_down_gives_bounded_503_never_anonymous_access (__main__.DependencyTest.test_postgres_down_gives_bounded_503_never_anonymous_access)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab4\student-tests\test_dependencies.py", line 64, in test_postgres_down_gives_bounded_503_never_anonymous_access
|
||||
r = call(method, path, body, token)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab4\student-tests\test_api.py", line 38, in call
|
||||
return status, hdrs, (json.loads(payload) if payload else None)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\AppData\Local\Programs\Python\Python311\Lib\json\__init__.py", line 346, in loads
|
||||
return _default_decoder.decode(s)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\AppData\Local\Programs\Python\Python311\Lib\json\decoder.py", line 337, in decode
|
||||
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\AppData\Local\Programs\Python\Python311\Lib\json\decoder.py", line 355, in raw_decode
|
||||
raise JSONDecodeError("Expecting value", s, err.value) from None
|
||||
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 9.323s
|
||||
|
||||
FAILED (errors=1)
|
||||
exit=1
|
||||
@@ -0,0 +1,116 @@
|
||||
$ python -u research/upload_memory.py --out evidence\recheck_2026-11-11
|
||||
размер файла: 8.7 MiB, sha256 9fcc26eea979
|
||||
{"UPLOAD_CONCURRENCY": 1, "statuses": {"201": 12}, "wall_seconds": 1.31, "peak_rss_mib": 99.2, "resource_limits": {"memory_bytes": 268435456, "nano_cpus": 1000000000}, "rss_before_mib": 93.3, "rss_after_mib": 99.2, "oom_killed": "false", "accepted_files_intact": "12/12", "p50_s": 0.83, "max_s": 1.3}
|
||||
{"UPLOAD_CONCURRENCY": 4, "statuses": {"201": 12}, "wall_seconds": 1.18, "peak_rss_mib": 119.0, "resource_limits": {"memory_bytes": 268435456, "nano_cpus": 1000000000}, "rss_before_mib": 93.5, "rss_after_mib": 119.0, "oom_killed": "false", "accepted_files_intact": "12/12", "p50_s": 0.87, "max_s": 1.17}
|
||||
{"UPLOAD_CONCURRENCY": 12, "statuses": {"201": 12}, "wall_seconds": 1.14, "peak_rss_mib": 121.2, "resource_limits": {"memory_bytes": 268435456, "nano_cpus": 1000000000}, "rss_before_mib": 93.4, "rss_after_mib": 121.2, "oom_killed": "false", "accepted_files_intact": "12/12", "p50_s": 0.94, "max_s": 1.13}
|
||||
exit=0
|
||||
|
||||
# Reconciliation rerun after disabling stale HTTP keep-alive reuse; memory results above retained.
|
||||
$ python -u research/reconcile_bench.py --out evidence\recheck_2026-11-11 --total 3000
|
||||
{"db_rows": 19554, "s3_objects": 19530, "timing_ms": {"db_query": 20.2, "s3_listing": 3540.6, "total": 3649.4}, "upload_batch_seconds": 2.3}
|
||||
{"db_rows": 20554, "s3_objects": 20530, "timing_ms": {"db_query": 19.0, "s3_listing": 3697.2, "total": 3806.4}, "upload_batch_seconds": 2.2}
|
||||
{"db_rows": 21554, "s3_objects": 21530, "timing_ms": {"db_query": 21.2, "s3_listing": 3939.1, "total": 4055.0}, "upload_batch_seconds": 2.1}
|
||||
{
|
||||
"date": "2026-11-11 18:21:28",
|
||||
"cost_of_scan": [
|
||||
{
|
||||
"db_rows": 19554,
|
||||
"s3_objects": 19530,
|
||||
"timing_ms": {
|
||||
"db_query": 20.2,
|
||||
"s3_listing": 3540.6,
|
||||
"total": 3649.4
|
||||
},
|
||||
"upload_batch_seconds": 2.3
|
||||
},
|
||||
{
|
||||
"db_rows": 20554,
|
||||
"s3_objects": 20530,
|
||||
"timing_ms": {
|
||||
"db_query": 19.0,
|
||||
"s3_listing": 3697.2,
|
||||
"total": 3806.4
|
||||
},
|
||||
"upload_batch_seconds": 2.2
|
||||
},
|
||||
{
|
||||
"db_rows": 21554,
|
||||
"s3_objects": 21530,
|
||||
"timing_ms": {
|
||||
"db_query": 21.2,
|
||||
"s3_listing": 3939.1,
|
||||
"total": 4055.0
|
||||
},
|
||||
"upload_batch_seconds": 2.1
|
||||
}
|
||||
],
|
||||
"injected": {
|
||||
"orphan_objects": 20,
|
||||
"missing_objects": 20,
|
||||
"size_mismatch": 5
|
||||
},
|
||||
"dry_run": {
|
||||
"orphan_objects_count": 20,
|
||||
"missing_objects_count": 20,
|
||||
"size_mismatch_count": 5,
|
||||
"orphans_skipped_fresh": 0,
|
||||
"timing_ms": {
|
||||
"db_query": 18.7,
|
||||
"s3_listing": 3652.1,
|
||||
"total": 3761.7
|
||||
},
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
}
|
||||
},
|
||||
"dry_run_repeatable": true,
|
||||
"dry_run_found_exactly_injected": true,
|
||||
"apply": {
|
||||
"fixed": {
|
||||
"deleted_orphans": 20,
|
||||
"marked_failed": 25
|
||||
},
|
||||
"orphans_skipped_fresh": 0,
|
||||
"timing_ms": {
|
||||
"db_query": 19.1,
|
||||
"s3_listing": 3859.1,
|
||||
"total": 4048.6
|
||||
},
|
||||
"orphan_objects_count": 20,
|
||||
"missing_objects_count": 20,
|
||||
"size_mismatch_count": 5
|
||||
},
|
||||
"fresh_uploads_during_apply": {
|
||||
"uploaded": 200,
|
||||
"intact": 200
|
||||
},
|
||||
"in_use_attachments_intact": "200/200",
|
||||
"message_bindings": {
|
||||
"messages": 20,
|
||||
"attachments": 200,
|
||||
"unchanged": true
|
||||
},
|
||||
"second_apply_idempotent": {
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
},
|
||||
"orphan_objects_count": 0,
|
||||
"missing_objects_count": 0
|
||||
},
|
||||
"marked_failed": [
|
||||
"object_missing=20",
|
||||
"size_mismatch=5"
|
||||
]
|
||||
}
|
||||
exit=0
|
||||
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,42 @@
|
||||
$ python -u research/upload_memory.py --out evidence\recheck_2026-11-11
|
||||
размер файла: 8.7 MiB, sha256 9fcc26eea979
|
||||
{"UPLOAD_CONCURRENCY": 1, "statuses": {"201": 12}, "wall_seconds": 1.31, "peak_rss_mib": 99.2, "resource_limits": {"memory_bytes": 268435456, "nano_cpus": 1000000000}, "rss_before_mib": 93.3, "rss_after_mib": 99.2, "oom_killed": "false", "accepted_files_intact": "12/12", "p50_s": 0.83, "max_s": 1.3}
|
||||
{"UPLOAD_CONCURRENCY": 4, "statuses": {"201": 12}, "wall_seconds": 1.18, "peak_rss_mib": 119.0, "resource_limits": {"memory_bytes": 268435456, "nano_cpus": 1000000000}, "rss_before_mib": 93.5, "rss_after_mib": 119.0, "oom_killed": "false", "accepted_files_intact": "12/12", "p50_s": 0.87, "max_s": 1.17}
|
||||
{"UPLOAD_CONCURRENCY": 12, "statuses": {"201": 12}, "wall_seconds": 1.14, "peak_rss_mib": 121.2, "resource_limits": {"memory_bytes": 268435456, "nano_cpus": 1000000000}, "rss_before_mib": 93.4, "rss_after_mib": 121.2, "oom_killed": "false", "accepted_files_intact": "12/12", "p50_s": 0.94, "max_s": 1.13}
|
||||
exit=0
|
||||
|
||||
$ python -u research/reconcile_bench.py --out evidence\recheck_2026-11-11 --total 3000
|
||||
{"db_rows": 16554, "s3_objects": 16530, "timing_ms": {"db_query": 17.7, "s3_listing": 2891.3, "total": 2998.3}, "upload_batch_seconds": 2.6}
|
||||
{"db_rows": 17554, "s3_objects": 17530, "timing_ms": {"db_query": 18.4, "s3_listing": 3280.3, "total": 3383.1}, "upload_batch_seconds": 2.2}
|
||||
{"db_rows": 18554, "s3_objects": 18530, "timing_ms": {"db_query": 16.8, "s3_listing": 3647.2, "total": 3754.3}, "upload_batch_seconds": 2.0}
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab4\research\reconcile_bench.py", line 204, in <module>
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
File "C:\Users\kolya5544\AppData\Local\Programs\Python\Python311\Lib\asyncio\runners.py", line 190, in run
|
||||
return runner.run(main)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\AppData\Local\Programs\Python\Python311\Lib\asyncio\runners.py", line 118, in run
|
||||
return self._loop.run_until_complete(task)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\AppData\Local\Programs\Python\Python311\Lib\asyncio\base_events.py", line 654, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab4\research\reconcile_bench.py", line 150, in main
|
||||
await bind_to_messages(s, token, chat, used_ids)
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab4\research\reconcile_bench.py", line 114, in bind_to_messages
|
||||
async with s.post("%s/api/v1/chats/%s/messages" % (BASE, chat),
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\tools\.venv\Lib\site-packages\aiohttp\client.py", line 1693, in __aenter__
|
||||
self._resp: _RetType_co = await self._coro
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\tools\.venv\Lib\site-packages\aiohttp\client.py", line 858, in _request
|
||||
resp = await handler(req)
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\tools\.venv\Lib\site-packages\aiohttp\client.py", line 836, in _connect_and_send_request
|
||||
await resp.start(conn)
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\tools\.venv\Lib\site-packages\aiohttp\client_reqrep.py", line 558, in start
|
||||
message, payload = await protocol.read() # type: ignore[union-attr]
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\kolya5544\source\repos\vmdMTUCI\tools\.venv\Lib\site-packages\aiohttp\streams.py", line 705, in read
|
||||
await self._waiter
|
||||
aiohttp.client_exceptions.ServerDisconnectedError: Server disconnected
|
||||
exit=1
|
||||
@@ -0,0 +1,116 @@
|
||||
{
|
||||
"before": {
|
||||
"mtusi-lab4-api-4": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-11T15:19:40.270198114Z",
|
||||
"health": "healthy",
|
||||
"volumes": [],
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-api-5": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-11T15:19:40.450834393Z",
|
||||
"health": "healthy",
|
||||
"volumes": [],
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-11T15:17:05.882305757Z",
|
||||
"health": "healthy",
|
||||
"volumes": [],
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-11T15:18:29.832733162Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab4_pgdata"
|
||||
],
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-11T15:17:06.948359559Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab4_seaweeddata"
|
||||
],
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-11T15:18:32.43836843Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab4_valkeydata"
|
||||
],
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"mtusi-lab4-api-4": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-11T15:22:57.324421301Z",
|
||||
"health": "healthy",
|
||||
"volumes": [],
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-api-5": {
|
||||
"service": "api",
|
||||
"started_at": "2026-11-11T15:22:57.482181899Z",
|
||||
"health": "healthy",
|
||||
"volumes": [],
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-lb-1": {
|
||||
"service": "lb",
|
||||
"started_at": "2026-11-11T15:22:53.438585878Z",
|
||||
"health": "healthy",
|
||||
"volumes": [],
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-postgres-1": {
|
||||
"service": "postgres",
|
||||
"started_at": "2026-11-11T15:22:53.862285534Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab4_pgdata"
|
||||
],
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-seaweedfs-1": {
|
||||
"service": "seaweedfs",
|
||||
"started_at": "2026-11-11T15:22:54.073709276Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab4_seaweeddata"
|
||||
],
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"mtusi-lab4-valkey-1": {
|
||||
"service": "valkey",
|
||||
"started_at": "2026-11-11T15:22:53.5998755Z",
|
||||
"health": "healthy",
|
||||
"volumes": [
|
||||
"mtusi-lab4_valkeydata"
|
||||
],
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000
|
||||
}
|
||||
},
|
||||
"all_services_restarted": true,
|
||||
"volumes_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"date": "2026-11-11",
|
||||
"base_solution_commit": "f875e2977f6592e1af6228dfe2138a769c327b08",
|
||||
"branch": "solution/bpi2302-kushnarenko",
|
||||
"python_client": "3.11.9 (tags/v3.11.9:de54cf5, Apr 2 2024, 10:12:12) [MSC v.1938 64 bit (AMD64)]",
|
||||
"source_sha256": {
|
||||
"Dockerfile": "f1dc711cacf00e3e2bd6b4a0616c925e357446ebda370e7e5fdb3170eaecd4cb",
|
||||
"compose.yaml": "e7f52308ad5f0f55bf1d61b83275bad2fb80be9f97b695cdd659d0d878cdd6a5",
|
||||
"infra/README.md": "33a4f7b870fd0b2bed98d46daf26d45dd7436176739672925246f8e3fe7c2278",
|
||||
"infra/haproxy/haproxy.cfg": "511fb24b32740c332cd7b6ea42d3f1cf252e1452de754ff97c8e6886f3692061",
|
||||
"requirements.txt": "709710dfd92af5f023ef9909488e9a14a7cacb62c67d67fff261d34ae4b779a0",
|
||||
"research/failover_sequence.py": "5cf120dfc9c79756a7bc913c900702ee597cc89ed4d7e21471a784a75b752ca4",
|
||||
"research/loadgen.py": "ec758237e5c57efe405fddac13ed80aab2ea31ff400761a951f89a31d7769b8b",
|
||||
"research/reconcile_bench.py": "6a6814f925b9e36621f6f7237473ad6b085b94e2717c2be79313cef7c8906a5a",
|
||||
"research/rolling_update.py": "447da4a9e2161bb232669f0558d6cb15ae3f0ef181fdf563e19cd56135b58240",
|
||||
"research/upload_memory.py": "07784bde3bf9cd15f586025adf6a37a2c3413ee756e099034c23cc789fbe7df1",
|
||||
"scripts/check_contract.py": "61e300fe5c1074360418e24b4653b12f96808247a2beaf2a64859aa360f81493",
|
||||
"scripts/init-secrets.sh": "7df8c218002a46ccb545d96051a6b958cf8b4dee14d94299a85984576b483f5b",
|
||||
"scripts/restart.cmd": "78e2689f97b3d0aafa72a0352a4a8cce2f41d78684c34042711e0af38e7cad19",
|
||||
"scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b",
|
||||
"scripts/restart.sh": "c57ad938ecaca510120296e9406918e6ef9ed24eb1b6dff16cf1b4d95c787e41",
|
||||
"scripts/restore-replicas.sh": "f55116712bb7f511bfc756507c3574a411256a41ea7ee6ba473c610ea5d1184b",
|
||||
"scripts/rolling-update.sh": "465791a134b48738a3e8d59819762159cc173d9332c6de9e2afcb583a4674559",
|
||||
"scripts/stop-one.cmd": "1955b8b0c879ea50d8efeb7fb43e8497356bb45482a91f90bd81a9c3fececdbf",
|
||||
"scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811",
|
||||
"scripts/stop-one.sh": "897cd2b56e6481cca7ac82378a0c071791f1e0e395d6a2e8d2a9f8e8faf50b38",
|
||||
"scripts/validate_openapi.py": "cd7a33221b1860c5d2330e52afa6cb222d375db671ae9ab379ba51a0d3a77946",
|
||||
"src/app/attachments.py": "430cf9672744a2192a72e9a9b64ab968510acb660590fd818b5f6dc6c3128ee0",
|
||||
"src/app/config.py": "2d0eb339e9b9caf9c69c1b497ba233758f9567c79e2edf76eb5facda3314fd38",
|
||||
"src/app/db.py": "7510882b0e8c9f6a24d680a156c285bc1a0fcf021473ca47173aa0744d845a78",
|
||||
"src/app/images.py": "3dc7434cbfbc9d4c01b95eb897c1332b2a2748dd9df88c99a7086f3e4cdf4c41",
|
||||
"src/app/main.py": "355aec75a78eaa7a8696684b251221a16c0828533fe1dbcfb30ab8cd04ca9b5c",
|
||||
"src/app/migrate.py": "cabc192ca4182af780636415a008d2dd4636dad7d3cf0f4a8a2b52a48aedaa7e",
|
||||
"src/app/passwords.py": "836fc282093058f6717eaa6374775251ed655dcd517f325dc60323a88154f261",
|
||||
"src/app/reconcile.py": "dd795a9d53a32bf2c107d2c053c170f7c4c7731b99aff57747af1edf49883a51",
|
||||
"src/app/s3.py": "d31dec589dd96607b3fa7fb5fa6013e3882638eb536a737dc766013209ce6247",
|
||||
"src/app/s3_bootstrap.py": "216d8121d6e7b25d95b845f4deaff18fd31eeb28484e51fe4c463c646a3af0d1",
|
||||
"src/app/serve.py": "3ebe12cf8cb2dddfc7d7234bba9bd190d868cc88d02e61fe640ba51dcf115b55",
|
||||
"src/app/sessions.py": "f975eb27c50c024f3e562f755ee97bb7bfe21cf5e01cc1d347bf107971c30670",
|
||||
"src/app/web.py": "23271be669a19bd7abfd6eaae25e25ce214718f0f623406cb9a2f6e77e365b83",
|
||||
"student-tests/fixtures/sample.jpg": "33a614b40fe6ae25fdbd99af22e7d9d3e5dbea455b36aac750de112c034a6311",
|
||||
"student-tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"student-tests/test_api.py": "2635113842a5b26ec1fdd98ea42fc861edb1d10895e8be6c2eff46ed73fdba9d",
|
||||
"student-tests/test_attachments.py": "90128b76736336bb3c37746549c951cbd853b93084b1d1c9605fffb3b77ffbbb",
|
||||
"student-tests/test_dependencies.py": "e152b87441d1374ce9b69d3d441d262e47890313d2ccc798f5e2cff2e3719399",
|
||||
"student-tests/test_expiry.py": "16a374ebe3df15c50c07a9fc3bc801c6c532dde3c4185b164c6aea4605923f83",
|
||||
"student-tests/test_idempotency.py": "4ee985a3ef26fefa816fdb136fa2b0644cca13d73fe3a226d1ba975a55698b5b",
|
||||
"student-tests/test_s3_failures.py": "81f8b7acbde19700c87e8d65af1803c071e9b0549d10788ae1f53729a64be175",
|
||||
"student-tests/test_topology.py": "576ff231fad871fa05cf6f14c6544c32389961f204b9dec8d2ecf250da15e157"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"date": "2026-11-11 18:21:28",
|
||||
"cost_of_scan": [
|
||||
{
|
||||
"db_rows": 19554,
|
||||
"s3_objects": 19530,
|
||||
"timing_ms": {
|
||||
"db_query": 20.2,
|
||||
"s3_listing": 3540.6,
|
||||
"total": 3649.4
|
||||
},
|
||||
"upload_batch_seconds": 2.3
|
||||
},
|
||||
{
|
||||
"db_rows": 20554,
|
||||
"s3_objects": 20530,
|
||||
"timing_ms": {
|
||||
"db_query": 19.0,
|
||||
"s3_listing": 3697.2,
|
||||
"total": 3806.4
|
||||
},
|
||||
"upload_batch_seconds": 2.2
|
||||
},
|
||||
{
|
||||
"db_rows": 21554,
|
||||
"s3_objects": 21530,
|
||||
"timing_ms": {
|
||||
"db_query": 21.2,
|
||||
"s3_listing": 3939.1,
|
||||
"total": 4055.0
|
||||
},
|
||||
"upload_batch_seconds": 2.1
|
||||
}
|
||||
],
|
||||
"args": {
|
||||
"out": "evidence\\recheck_2026-11-11",
|
||||
"total": 3000
|
||||
},
|
||||
"injected": {
|
||||
"orphan_objects": 20,
|
||||
"missing_objects": 20,
|
||||
"size_mismatch": 5
|
||||
},
|
||||
"dry_run": {
|
||||
"orphan_objects_count": 20,
|
||||
"missing_objects_count": 20,
|
||||
"size_mismatch_count": 5,
|
||||
"orphans_skipped_fresh": 0,
|
||||
"timing_ms": {
|
||||
"db_query": 18.7,
|
||||
"s3_listing": 3652.1,
|
||||
"total": 3761.7
|
||||
},
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
}
|
||||
},
|
||||
"dry_run_repeatable": true,
|
||||
"dry_run_found_exactly_injected": true,
|
||||
"apply": {
|
||||
"fixed": {
|
||||
"deleted_orphans": 20,
|
||||
"marked_failed": 25
|
||||
},
|
||||
"orphans_skipped_fresh": 0,
|
||||
"timing_ms": {
|
||||
"db_query": 19.1,
|
||||
"s3_listing": 3859.1,
|
||||
"total": 4048.6
|
||||
},
|
||||
"orphan_objects_count": 20,
|
||||
"missing_objects_count": 20,
|
||||
"size_mismatch_count": 5
|
||||
},
|
||||
"fresh_uploads_during_apply": {
|
||||
"uploaded": 200,
|
||||
"intact": 200
|
||||
},
|
||||
"in_use_attachments_intact": "200/200",
|
||||
"message_bindings": {
|
||||
"messages": 20,
|
||||
"attachments": 200,
|
||||
"unchanged": true
|
||||
},
|
||||
"second_apply_idempotent": {
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
},
|
||||
"orphan_objects_count": 0,
|
||||
"missing_objects_count": 0
|
||||
},
|
||||
"marked_failed": [
|
||||
"object_missing=20",
|
||||
"size_mismatch=5"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"date": "2026-11-11 18:18:37",
|
||||
"file_mib": 8.73,
|
||||
"clients": 12,
|
||||
"api_memory_limit": "256m",
|
||||
"runs": [
|
||||
{
|
||||
"UPLOAD_CONCURRENCY": 1,
|
||||
"statuses": {
|
||||
"201": 12
|
||||
},
|
||||
"wall_seconds": 1.31,
|
||||
"peak_rss_mib": 99.2,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"rss_before_mib": 93.3,
|
||||
"rss_after_mib": 99.2,
|
||||
"oom_killed": "false",
|
||||
"accepted_files_intact": "12/12",
|
||||
"p50_s": 0.83,
|
||||
"max_s": 1.3
|
||||
},
|
||||
{
|
||||
"UPLOAD_CONCURRENCY": 4,
|
||||
"statuses": {
|
||||
"201": 12
|
||||
},
|
||||
"wall_seconds": 1.18,
|
||||
"peak_rss_mib": 119.0,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"rss_before_mib": 93.5,
|
||||
"rss_after_mib": 119.0,
|
||||
"oom_killed": "false",
|
||||
"accepted_files_intact": "12/12",
|
||||
"p50_s": 0.87,
|
||||
"max_s": 1.17
|
||||
},
|
||||
{
|
||||
"UPLOAD_CONCURRENCY": 12,
|
||||
"statuses": {
|
||||
"201": 12
|
||||
},
|
||||
"wall_seconds": 1.14,
|
||||
"peak_rss_mib": 121.2,
|
||||
"resource_limits": {
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
"rss_before_mib": 93.4,
|
||||
"rss_after_mib": 121.2,
|
||||
"oom_killed": "false",
|
||||
"accepted_files_intact": "12/12",
|
||||
"p50_s": 0.94,
|
||||
"max_s": 1.13
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
{
|
||||
"date": "2026-10-03 02:38:10",
|
||||
"cost_of_scan": [
|
||||
{
|
||||
"db_rows": 13271,
|
||||
"s3_objects": 13270,
|
||||
"timing_ms": {
|
||||
"db_query": 12.6,
|
||||
"s3_listing": 2073.5,
|
||||
"total": 2168.0
|
||||
},
|
||||
"upload_batch_seconds": 1.6
|
||||
},
|
||||
{
|
||||
"db_rows": 14271,
|
||||
"s3_objects": 14270,
|
||||
"timing_ms": {
|
||||
"db_query": 13.2,
|
||||
"s3_listing": 2241.4,
|
||||
"total": 2341.3
|
||||
},
|
||||
"upload_batch_seconds": 1.7
|
||||
},
|
||||
{
|
||||
"db_rows": 15271,
|
||||
"s3_objects": 15270,
|
||||
"timing_ms": {
|
||||
"db_query": 13.7,
|
||||
"s3_listing": 2401.3,
|
||||
"total": 2497.4
|
||||
},
|
||||
"upload_batch_seconds": 1.7
|
||||
}
|
||||
],
|
||||
"args": {
|
||||
"out": "evidence",
|
||||
"total": 3000
|
||||
},
|
||||
"injected": {
|
||||
"orphan_objects": 20,
|
||||
"missing_objects": 20,
|
||||
"size_mismatch": 5
|
||||
},
|
||||
"dry_run": {
|
||||
"orphan_objects_count": 20,
|
||||
"missing_objects_count": 20,
|
||||
"size_mismatch_count": 5,
|
||||
"orphans_skipped_fresh": 0,
|
||||
"timing_ms": {
|
||||
"db_query": 12.6,
|
||||
"s3_listing": 2408.1,
|
||||
"total": 2505.2
|
||||
},
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
}
|
||||
},
|
||||
"dry_run_repeatable": true,
|
||||
"dry_run_found_exactly_injected": true,
|
||||
"apply": {
|
||||
"fixed": {
|
||||
"deleted_orphans": 20,
|
||||
"marked_failed": 25
|
||||
},
|
||||
"orphans_skipped_fresh": 0,
|
||||
"timing_ms": {
|
||||
"db_query": 14.0,
|
||||
"s3_listing": 2429.5,
|
||||
"total": 2606.4
|
||||
},
|
||||
"orphan_objects_count": 20,
|
||||
"missing_objects_count": 20,
|
||||
"size_mismatch_count": 5
|
||||
},
|
||||
"fresh_uploads_during_apply": {
|
||||
"uploaded": 200,
|
||||
"intact": 200
|
||||
},
|
||||
"in_use_attachments_intact": "200/200",
|
||||
"second_apply_idempotent": {
|
||||
"fixed": {
|
||||
"deleted_orphans": 0,
|
||||
"marked_failed": 0
|
||||
},
|
||||
"orphan_objects_count": 0,
|
||||
"missing_objects_count": 0
|
||||
},
|
||||
"marked_failed": [
|
||||
"object_missing=20",
|
||||
"size_mismatch=5"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"date": "2026-10-03 02:33:53",
|
||||
"file_mib": 8.73,
|
||||
"clients": 12,
|
||||
"api_memory_limit": "256m",
|
||||
"runs": [
|
||||
{
|
||||
"UPLOAD_CONCURRENCY": 1,
|
||||
"statuses": {
|
||||
"201": 12
|
||||
},
|
||||
"wall_seconds": 1.35,
|
||||
"peak_rss_mib": 99.3,
|
||||
"rss_before_mib": 93.4,
|
||||
"rss_after_mib": 99.3,
|
||||
"oom_killed": "false",
|
||||
"accepted_files_intact": "12/12",
|
||||
"p50_s": 0.89,
|
||||
"max_s": 1.35
|
||||
},
|
||||
{
|
||||
"UPLOAD_CONCURRENCY": 4,
|
||||
"statuses": {
|
||||
"201": 12
|
||||
},
|
||||
"wall_seconds": 1.0,
|
||||
"peak_rss_mib": 116.1,
|
||||
"rss_before_mib": 93.9,
|
||||
"rss_after_mib": 116.1,
|
||||
"oom_killed": "false",
|
||||
"accepted_files_intact": "12/12",
|
||||
"p50_s": 0.79,
|
||||
"max_s": 0.99
|
||||
},
|
||||
{
|
||||
"UPLOAD_CONCURRENCY": 12,
|
||||
"statuses": {
|
||||
"201": 12
|
||||
},
|
||||
"wall_seconds": 1.05,
|
||||
"peak_rss_mib": 122.6,
|
||||
"rss_before_mib": 93.6,
|
||||
"rss_after_mib": 122.6,
|
||||
"oom_killed": "false",
|
||||
"accepted_files_intact": "12/12",
|
||||
"p50_s": 0.82,
|
||||
"max_s": 1.04
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
# Балансировщик lab3: HAProxy 3.0. Единственная публичная точка входа; API-реплики наружу не опубликованы.
|
||||
global
|
||||
log stdout format raw local0 notice
|
||||
maxconn 4096
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option dontlognull
|
||||
option forwardfor
|
||||
# connect короткий: недоступная реплика быстро заменяется другой (redispatch)
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 5s
|
||||
retries 3
|
||||
option redispatch 1
|
||||
# Only retry before a connection is established: upload POST has no idempotency key.
|
||||
retry-on conn-failure
|
||||
|
||||
# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров).
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold nx 2s
|
||||
hold other 2s
|
||||
hold obsolete 2s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
# If the shared DB is down, live API processes still produce contract JSON,
|
||||
# validate missing credentials, and serve /health/live independently of DB.
|
||||
use_backend api_live if { nbsrv(api) eq 0 }
|
||||
default_backend api
|
||||
|
||||
# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется.
|
||||
frontend lb_health
|
||||
bind :8081
|
||||
monitor-uri /healthz
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
# Проверка готовности каждые 1 с; 2 неудачи подряд -> сервер исключён (обнаружение отказа <= ~3 с), 2 успеха -> возвращён.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 1-8 api:8080
|
||||
|
||||
backend api_live
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/live
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 1-8 api:8080
|
||||
@@ -0,0 +1,52 @@
|
||||
-- 0001: пользователи, чаты, участники, сообщения, сессии.
|
||||
-- Источник истины для всех данных — PostgreSQL. Каждая миграция выполняется в транзакции.
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
-- Уникальность username: гонка двух регистраций решается ограничением БД, а не проверкой в приложении.
|
||||
CREATE UNIQUE INDEX users_username_key ON users (username);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
-- chat_created_at денормализован ради keyset-пагинации списка чатов пользователя без join+sort.
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
chat_created_at timestamptz NOT NULL,
|
||||
ord smallint NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_chats ON chat_members (user_id, chat_created_at, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users (id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000)
|
||||
);
|
||||
-- История чата: WHERE chat_id = ? AND (created_at, id) > (?, ?) ORDER BY created_at, id LIMIT n
|
||||
-- обслуживается этим индексом без сортировки и без сканирования всей истории.
|
||||
CREATE INDEX messages_chat_history ON messages (chat_id, created_at, id);
|
||||
|
||||
-- Серверные сессии. В БД хранится только SHA-256 токена: утечка БД не раскрывает рабочие токены.
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
CREATE INDEX sessions_expires_at ON sessions (expires_at);
|
||||
@@ -0,0 +1,17 @@
|
||||
-- 0002: общая идемпотентность отправки сообщений (lab3).
|
||||
-- Область уникальности: пользователь + чат + ключ (метод и путь фиксированы: POST /chats/{id}/messages).
|
||||
-- Запись ключа и сообщения — одна транзакция: побочный эффект и результат фиксируются атомарно.
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
idem_key text NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, idem_key),
|
||||
CONSTRAINT idem_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
-- сообщение вставляется после ключа в той же транзакции -> проверка внешнего ключа откладывается до COMMIT
|
||||
CONSTRAINT idem_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) ON DELETE CASCADE
|
||||
DEFERRABLE INITIALLY DEFERRED
|
||||
);
|
||||
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,35 @@
|
||||
-- 0003: отправка сообщения одной серверной функцией (оптимизация по итогам трека 1: меньше обращений к БД на запрос).
|
||||
-- Семантика идентична версии «несколько запросов в транзакции» (migrations 0002 + код приложения):
|
||||
-- проверка членства -> просроченный ключ удаляется -> INSERT ключа (ON CONFLICT DO NOTHING) -> INSERT сообщения.
|
||||
-- Функция выполняется атомарно (один вызов = одна транзакция); гонку одинаковых ключей по-прежнему решает уникальный индекс.
|
||||
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision)
|
||||
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
|
||||
request_hash bytea)
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE
|
||||
v_new uuid := gen_random_uuid();
|
||||
v_ins uuid;
|
||||
v_created timestamptz;
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
|
||||
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz, NULL::bytea;
|
||||
RETURN;
|
||||
END IF;
|
||||
DELETE FROM idempotency_keys k
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
|
||||
AND k.created_at < now() - make_interval(secs => p_ttl);
|
||||
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, v_new)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING message_id INTO v_ins;
|
||||
IF v_ins IS NOT NULL THEN
|
||||
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
|
||||
RETURNING created_at INTO v_created;
|
||||
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash;
|
||||
ELSE
|
||||
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,90 @@
|
||||
-- 0004 (lab4): метаданные вложений в PostgreSQL, байты — в приватном S3-бакете.
|
||||
-- Ключ объекта назначает сервер (att/<id>/original); filename — только отображаемое имя.
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users (id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
status text NOT NULL DEFAULT 'ready',
|
||||
error_code text,
|
||||
s3_key text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255)
|
||||
);
|
||||
CREATE UNIQUE INDEX attachments_s3_key_key ON attachments (s3_key);
|
||||
CREATE INDEX attachments_chat ON attachments (chat_id);
|
||||
|
||||
-- Связь сообщение <-> вложения (порядок сохраняется). Вложение может быть привязано к нескольким сообщениям чата.
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id);
|
||||
|
||||
-- Отправка сообщения с вложениями: заменяет функцию из 0003 (добавлен параметр p_atts и результат msg_atts).
|
||||
DROP FUNCTION send_message(uuid, uuid, text, bytea, text, double precision);
|
||||
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision,
|
||||
p_atts uuid[])
|
||||
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
|
||||
request_hash bytea, msg_atts uuid[])
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE
|
||||
v_new uuid := gen_random_uuid();
|
||||
v_ins uuid;
|
||||
v_created timestamptz;
|
||||
v_found int;
|
||||
v_failed int;
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
|
||||
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz,
|
||||
NULL::bytea, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
IF cardinality(p_atts) > 0 THEN
|
||||
SELECT count(*), count(*) FILTER (WHERE a.status = 'failed') INTO v_found, v_failed
|
||||
FROM attachments a WHERE a.id = ANY (p_atts) AND a.chat_id = p_chat;
|
||||
IF v_found <> cardinality(p_atts) THEN
|
||||
RETURN QUERY SELECT 'attachment_not_found'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
|
||||
NULL::timestamptz, NULL::bytea, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
IF v_failed > 0 THEN
|
||||
RETURN QUERY SELECT 'attachment_failed'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
|
||||
NULL::timestamptz, NULL::bytea, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
DELETE FROM idempotency_keys k
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
|
||||
AND k.created_at < now() - make_interval(secs => p_ttl);
|
||||
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, v_new)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING message_id INTO v_ins;
|
||||
IF v_ins IS NOT NULL THEN
|
||||
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
|
||||
RETURNING created_at INTO v_created;
|
||||
INSERT INTO message_attachments (message_id, attachment_id, position)
|
||||
SELECT v_new, a, (o - 1)::smallint FROM unnest(p_atts) WITH ORDINALITY AS t(a, o);
|
||||
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash, p_atts;
|
||||
ELSE
|
||||
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash,
|
||||
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[])
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
@@ -0,0 +1,30 @@
|
||||
aiobotocore==3.9.2
|
||||
aiohappyeyeballs==2.7.1
|
||||
aiohttp==3.14.3
|
||||
aioitertools==0.13.0
|
||||
aiosignal==1.4.0
|
||||
argon2-cffi==23.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
asyncpg==0.30.0
|
||||
attrs==26.1.0
|
||||
botocore==1.43.106
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
frozenlist==1.8.0
|
||||
h11==0.16.0
|
||||
httptools==0.8.0
|
||||
idna==3.20
|
||||
jmespath==1.1.0
|
||||
multidict==6.9.1
|
||||
pillow==12.3.0
|
||||
propcache==0.5.4
|
||||
pycparser==3.0
|
||||
python-dateutil==2.9.0.post0
|
||||
redis==5.2.1
|
||||
six==1.17.0
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.0
|
||||
uvloop==0.21.0; sys_platform != "win32"
|
||||
wrapt==2.5.0
|
||||
yarl==1.25.1
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей).
|
||||
|
||||
Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с):
|
||||
t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса)
|
||||
t=25 восстановить (scripts/restore-replicas.sh)
|
||||
t=40 остановить реплику B
|
||||
t=55 восстановить
|
||||
Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок
|
||||
(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей.
|
||||
|
||||
python research/failover_sequence.py --out evidence
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import loadgen # noqa: E402
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def sh(*args):
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def api_ids():
|
||||
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab4",
|
||||
"--filter", "label=com.docker.compose.service=api").split()
|
||||
|
||||
|
||||
def scenario(rec, plan):
|
||||
"""Выполняется в отдельном потоке: действия по расписанию относительно rec.t0."""
|
||||
for at, action in plan:
|
||||
time.sleep(max(0, rec.t0 + at - time.time()))
|
||||
t = time.time() - rec.t0
|
||||
if action == "stop":
|
||||
target = api_ids()[0]
|
||||
subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True)
|
||||
rec.marks.append({"t": round(t, 2), "action": "stop", "container": target})
|
||||
else:
|
||||
subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True)
|
||||
rec.marks.append({"t": round(t, 2), "action": "restore"})
|
||||
|
||||
|
||||
async def main(args):
|
||||
stand = loadgen.Stand(BASE)
|
||||
await stand.setup()
|
||||
rec = loadgen.Recorder()
|
||||
rec.marks = []
|
||||
plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")]
|
||||
threading.Thread(target=scenario, args=(rec, plan), daemon=True).start()
|
||||
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
|
||||
await asyncio.sleep(2)
|
||||
check = await loadgen.verify(stand, rec)
|
||||
stops = [m for m in rec.marks if m["action"] == "stop"]
|
||||
restores = [m for m in rec.marks if m["action"] == "restore"]
|
||||
windows = []
|
||||
for i, stop in enumerate(stops):
|
||||
t0 = stop["t"]
|
||||
errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10]
|
||||
last_err = max((e[0] for e in errs), default=t0)
|
||||
restore_t = restores[i]["t"] if i < len(restores) else t0 + 15
|
||||
windows.append({"stopped_container": stop["container"], "stop_at_s": t0,
|
||||
"rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"),
|
||||
"rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"),
|
||||
"rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"),
|
||||
"failed_attempts_within_10s": len(errs),
|
||||
"error_statuses": sorted({str(e[2]) for e in errs}),
|
||||
"seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0,
|
||||
"p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")})
|
||||
series = loadgen.per_second(rec)
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows,
|
||||
"total": loadgen.summarize(rec), "verification": check,
|
||||
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0,
|
||||
"timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])}
|
||||
for t, b in sorted(series.items())]}
|
||||
with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
plot(report, os.path.join(args.out, "failover_sequence.png"))
|
||||
print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def plot(report, path):
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
tl = report["timeline"]
|
||||
t = [x["t"] for x in tl]
|
||||
fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True)
|
||||
ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность")
|
||||
ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)")
|
||||
ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST")
|
||||
for a in ax:
|
||||
for m in report["marks"]:
|
||||
a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1)
|
||||
ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9)
|
||||
fig.tight_layout(); fig.savefig(path, dpi=110)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--seconds", type=int, default=70)
|
||||
ap.add_argument("--concurrency", type=int, default=16)
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Генератор нагрузки для экспериментов lab4+ (asyncio + aiohttp, closed-loop).
|
||||
|
||||
Каждый воркер выполняет цикл: POST сообщения с новым Idempotency-Key (90 %) или GET первой страницы истории (10 %).
|
||||
При ошибке соединения/5xx POST повторяется с ТЕМ ЖЕ ключом (до RETRIES раз) — именно так клиент безопасно
|
||||
повторяет неподтверждённую запись. Результат: по секундам успехи/ошибки/латентности, список подтверждённых
|
||||
сообщений (для сверки с историей), число повторов.
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import aiohttp
|
||||
|
||||
RETRIES = 3
|
||||
|
||||
|
||||
class Stand:
|
||||
"""Набор пользователей/чатов, созданных через API (через LB), для одного эксперимента."""
|
||||
|
||||
def __init__(self, base, users=20, chats=20):
|
||||
self.base, self.n_users, self.n_chats = base, users, chats
|
||||
self.users, self.chats = [], []
|
||||
|
||||
async def setup(self):
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as s:
|
||||
for i in range(self.n_users):
|
||||
name = "ld_%s" % uuid.uuid4().hex[:14]
|
||||
pw = "load-test-password-" + uuid.uuid4().hex[:8]
|
||||
async with s.post(self.base + "/api/v1/users", json={"username": name, "display_name": "load %d" % i, "password": pw}) as r:
|
||||
user = await r.json()
|
||||
assert r.status == 201, user
|
||||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||||
async with s.post(self.base + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||||
sess = await r.json()
|
||||
assert r.status == 201, sess
|
||||
self.users.append({"id": user["id"], "token": sess["token"]})
|
||||
for i in range(self.n_chats):
|
||||
members = random.sample(self.users, 3)
|
||||
owner = members[0]
|
||||
async with s.post(self.base + "/api/v1/chats", json={"title": "load %d" % i, "member_ids": [m["id"] for m in members]},
|
||||
headers={"Authorization": "Bearer " + owner["token"]}) as r:
|
||||
chat = await r.json()
|
||||
assert r.status == 201, chat
|
||||
self.chats.append({"id": chat["id"], "members": members})
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self):
|
||||
self.t0 = time.time()
|
||||
self.events = [] # (t, kind, status, latency_ms, instance)
|
||||
self.confirmed = [] # (chat_id, message_id, text)
|
||||
self.retries = 0
|
||||
self.final_failures = 0
|
||||
self.unconfirmed = [] # (chat_id, key, text) — финально не подтверждены (могли записаться!)
|
||||
|
||||
|
||||
async def worker(stand, rec, deadline_fn, read_ratio=0.1, text_size=0, think=0.0):
|
||||
connector = aiohttp.TCPConnector(limit=2)
|
||||
timeout = aiohttp.ClientTimeout(total=5)
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as s:
|
||||
n = 0
|
||||
while not deadline_fn():
|
||||
chat = random.choice(stand.chats)
|
||||
user = random.choice(chat["members"])
|
||||
hdr = {"Authorization": "Bearer " + user["token"]}
|
||||
if random.random() < read_ratio:
|
||||
t = time.perf_counter()
|
||||
try:
|
||||
async with s.get("%s/api/v1/chats/%s/messages?limit=20" % (stand.base, chat["id"]), headers=hdr) as r:
|
||||
await r.read()
|
||||
rec.events.append((time.time() - rec.t0, "GET", r.status, (time.perf_counter() - t) * 1000, r.headers.get("X-Instance-Id")))
|
||||
except Exception as exc:
|
||||
rec.events.append((time.time() - rec.t0, "GET", type(exc).__name__, (time.perf_counter() - t) * 1000, None))
|
||||
continue
|
||||
key = uuid.uuid4().hex
|
||||
text = "m-%s-%d" % (key[:8], n) + "я" * text_size
|
||||
n += 1
|
||||
for attempt in range(RETRIES + 1):
|
||||
t = time.perf_counter()
|
||||
try:
|
||||
async with s.post("%s/api/v1/chats/%s/messages" % (stand.base, chat["id"]), json={"text": text},
|
||||
headers=dict(hdr, **{"Idempotency-Key": key})) as r:
|
||||
body = await r.json(content_type=None)
|
||||
status, inst = r.status, r.headers.get("X-Instance-Id")
|
||||
except Exception as exc:
|
||||
status, inst, body = type(exc).__name__, None, None
|
||||
rec.events.append((time.time() - rec.t0, "POST", status, (time.perf_counter() - t) * 1000, inst))
|
||||
if status == 201:
|
||||
rec.confirmed.append((chat["id"], body["id"], text))
|
||||
break
|
||||
if attempt < RETRIES and (not isinstance(status, int) or status >= 500 or status == 429):
|
||||
rec.retries += 1
|
||||
await asyncio.sleep(0.05 * (attempt + 1))
|
||||
continue
|
||||
rec.final_failures += 1
|
||||
rec.unconfirmed.append((chat["id"], key, text))
|
||||
break
|
||||
if think:
|
||||
await asyncio.sleep(think)
|
||||
|
||||
|
||||
async def run_load(stand, rec, seconds, concurrency, **kw):
|
||||
end = time.time() + seconds
|
||||
await asyncio.gather(*(worker(stand, rec, lambda: time.time() > end, **kw) for _ in range(concurrency)))
|
||||
|
||||
|
||||
async def verify(stand, rec):
|
||||
"""Сверка: каждое подтверждённое сообщение читается, дублей по тексту нет, неподтверждённые встречаются не более 1 раза."""
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
stored = {}
|
||||
async with aiohttp.ClientSession(timeout=timeout) as s:
|
||||
for chat in stand.chats:
|
||||
hdr = {"Authorization": "Bearer " + chat["members"][0]["token"]}
|
||||
cursor, items = None, []
|
||||
while True:
|
||||
url = "%s/api/v1/chats/%s/messages?limit=100%s" % (stand.base, chat["id"], "&cursor=" + cursor if cursor else "")
|
||||
async with s.get(url, headers=hdr) as r:
|
||||
page = await r.json()
|
||||
assert r.status == 200, page
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
stored[chat["id"]] = items
|
||||
ids = {m["id"] for items in stored.values() for m in items}
|
||||
texts = [m["text"] for items in stored.values() for m in items]
|
||||
confirmed_missing = [c for c in rec.confirmed if c[1] not in ids]
|
||||
dup_texts = len(texts) - len(set(texts))
|
||||
unconfirmed_present = sum(1 for c in rec.unconfirmed if c[2] in set(texts))
|
||||
return {"stored_messages": len(ids), "confirmed_201": len(rec.confirmed), "confirmed_missing_in_history": len(confirmed_missing),
|
||||
"duplicate_texts_in_history": dup_texts, "unconfirmed_requests": len(rec.unconfirmed),
|
||||
"unconfirmed_that_were_written": unconfirmed_present, "retries": rec.retries, "final_failures": rec.final_failures}
|
||||
|
||||
|
||||
def per_second(rec, kind="POST"):
|
||||
buckets = {}
|
||||
for t, k, status, lat, inst in rec.events:
|
||||
if k != kind:
|
||||
continue
|
||||
b = buckets.setdefault(int(t), {"ok": 0, "err": 0, "lat": [], "instances": set()})
|
||||
if status == 201 or (kind == "GET" and status == 200):
|
||||
b["ok"] += 1
|
||||
b["lat"].append(lat)
|
||||
if inst:
|
||||
b["instances"].add(inst)
|
||||
else:
|
||||
b["err"] += 1
|
||||
return buckets
|
||||
|
||||
|
||||
def pct(values, p):
|
||||
if not values:
|
||||
return None
|
||||
values = sorted(values)
|
||||
return round(values[max(0, int(len(values) * p + 0.9999) - 1)], 2)
|
||||
|
||||
|
||||
def summarize(rec, t_from=None, t_to=None, kind="POST"):
|
||||
rows = [e for e in rec.events if e[1] == kind and (t_from is None or e[0] >= t_from) and (t_to is None or e[0] < t_to)]
|
||||
if not rows:
|
||||
return {}
|
||||
dur = (t_to if t_to is not None else max(e[0] for e in rows)) - (t_from if t_from is not None else min(e[0] for e in rows))
|
||||
ok = [e[3] for e in rows if e[2] == 201 or (kind == "GET" and e[2] == 200)]
|
||||
errs = {}
|
||||
for e in rows:
|
||||
if not (e[2] == 201 or (kind == "GET" and e[2] == 200)):
|
||||
errs[str(e[2])] = errs.get(str(e[2]), 0) + 1
|
||||
return {"requests": len(rows), "ok": len(ok), "errors": errs, "seconds": round(dur, 2), "ok_rps": round(len(ok) / dur, 1) if dur > 0 else None,
|
||||
"error_ratio": round((len(rows) - len(ok)) / len(rows), 4), "p50_ms": pct(ok, .5), "p95_ms": pct(ok, .95), "p99_ms": pct(ok, .99)}
|
||||
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 2 (согласованность двух хранилищ): reconciler S3 <-> PostgreSQL — внедрение ошибок, dry-run/apply, стоимость обхода.
|
||||
|
||||
Сценарий (стенд запущен; используются синтетические данные):
|
||||
1. Наращиваем число вложений тремя порциями (до ~N) и после каждой порции замеряем стоимость dry-run обхода.
|
||||
2. Внедряем ошибки на ОБЕИХ сторонах: A) 20 «осиротевших» объектов в S3 (нет строки в БД);
|
||||
B) у 20 вложений удаляем объект в S3 (строка есть, объекта нет); C) у 5 вложений подменяем объект на другой по размеру.
|
||||
3. dry-run: находит ровно внедрённое и ничего не меняет (повтор даёт тот же отчёт);
|
||||
apply с grace=5 с ПАРАЛЛЕЛЬНО идущей загрузкой 200 новых вложений: удаляются только старые осиротевшие объекты;
|
||||
свежие/используемые вложения не затронуты (скачиваются, sha256 совпадает);
|
||||
повторный apply — идемпотентен (изменений нет).
|
||||
Результат: evidence/track2_reconcile.json
|
||||
python research/reconcile_bench.py --out evidence --total 3000
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import aiohttp
|
||||
from PIL import Image
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def sh(*args):
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True)
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return sh("docker", "compose", "--env-file", ".env", *args).stdout
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
out = compose("run", "--rm", "-T", "api", "python", "-m", "app.reconcile", "--max-listed", "100", *args)
|
||||
return json.loads(out[out.index("{"):])
|
||||
|
||||
|
||||
def in_api(code):
|
||||
r = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError("сбой во внедрении ошибок: " + r.stderr[-500:])
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def tiny_png(seed):
|
||||
img = Image.new("RGB", (40, 30), (seed % 256, (seed // 256) % 256, 90))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
async def register(s):
|
||||
name = "rc_%s" % uuid.uuid4().hex[:12]
|
||||
pw = "reconcile-test-" + uuid.uuid4().hex[:6]
|
||||
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "r", "password": pw}) as r:
|
||||
user = await r.json()
|
||||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||||
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||||
token = (await r.json())["token"]
|
||||
async with s.post(BASE + "/api/v1/chats", json={"title": "reconcile", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
|
||||
chat = await r.json()
|
||||
return token, chat["id"]
|
||||
|
||||
|
||||
async def upload_many(s, token, chat_id, n, offset=0, concurrency=16):
|
||||
ids, sem = [], asyncio.Semaphore(concurrency)
|
||||
|
||||
async def one(i):
|
||||
async with sem:
|
||||
data = tiny_png(offset + i)
|
||||
for attempt in range(4): # повтор при разрыве keep-alive соединения (гонка закрытия на стороне сервера)
|
||||
try:
|
||||
async with s.post("%s/api/v1/chats/%s/attachments?filename=r%d.png" % (BASE, chat_id, i), data=data,
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) as r:
|
||||
body = await r.json(content_type=None)
|
||||
if r.status == 201:
|
||||
ids.append((body["id"], body["sha256"]))
|
||||
break
|
||||
except aiohttp.ClientError:
|
||||
await asyncio.sleep(0.1)
|
||||
await asyncio.gather(*(one(i) for i in range(n)))
|
||||
return ids
|
||||
|
||||
|
||||
async def check_intact(s, token, items):
|
||||
ok = 0
|
||||
for att_id, sha in items:
|
||||
async with s.get("%s/api/v1/attachments/%s/content" % (BASE, att_id), headers={"Authorization": "Bearer " + token}) as r:
|
||||
if r.status == 200 and hashlib.sha256(await r.read()).hexdigest() == sha:
|
||||
ok += 1
|
||||
return ok
|
||||
|
||||
|
||||
def keys_for(ids):
|
||||
out = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c",
|
||||
"SELECT s3_key FROM attachments WHERE id IN (%s)" % ",".join("'%s'" % i for i in ids)).stdout.split()
|
||||
return out
|
||||
|
||||
|
||||
async def bind_to_messages(s, token, chat, ids):
|
||||
"""Create real message references, not just ready attachment metadata."""
|
||||
headers = {"Authorization": "Bearer " + token}
|
||||
for start in range(0, len(ids), 10):
|
||||
batch = ids[start:start + 10]
|
||||
async with s.post("%s/api/v1/chats/%s/messages" % (BASE, chat),
|
||||
json={"text": "reconciler keeper", "attachment_ids": batch},
|
||||
headers={**headers, "Idempotency-Key": str(uuid.uuid4())}) as r:
|
||||
body = await r.json()
|
||||
assert r.status == 201 and body["attachment_ids"] == batch, (r.status, body)
|
||||
|
||||
|
||||
async def message_bindings(s, token, chat):
|
||||
async with s.get("%s/api/v1/chats/%s/messages?limit=100" % (BASE, chat),
|
||||
headers={"Authorization": "Bearer " + token}) as r:
|
||||
r.raise_for_status()
|
||||
body = await r.json()
|
||||
return {m["id"]: m["attachment_ids"] for m in body["items"]}
|
||||
|
||||
|
||||
async def main(args):
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "cost_of_scan": [], "args": vars(args)}
|
||||
# Docker commands block the event loop longer than the server's keep-alive.
|
||||
# Do not reuse idle HTTP sockets after a scan or fault-injection command.
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60),
|
||||
connector=aiohttp.TCPConnector(limit=32, force_close=True)) as s:
|
||||
token, chat = await register(s)
|
||||
base = reconcile("--grace", "0", "--apply") # нормализация: убираем следы прошлых прогонов
|
||||
base = reconcile("--grace", "0")
|
||||
assert (base["orphan_objects_count"], base["missing_objects_count"], base["size_mismatch_count"]) == (0, 0, 0), base
|
||||
uploaded, step = [], args.total // 3
|
||||
for part in range(3):
|
||||
t = time.perf_counter()
|
||||
uploaded += await upload_many(s, token, chat, step, offset=len(uploaded))
|
||||
up_s = time.perf_counter() - t
|
||||
r = reconcile("--grace", "900")
|
||||
report["cost_of_scan"].append({"db_rows": r["db_rows"], "s3_objects": r["s3_objects"], "timing_ms": r["timing_ms"],
|
||||
"upload_batch_seconds": round(up_s, 1)})
|
||||
print(json.dumps(report["cost_of_scan"][-1]))
|
||||
# --- внедрение ошибок ---
|
||||
sample = random.sample(uploaded, 25)
|
||||
missing_ids = [a for a, _ in sample[:20]]
|
||||
mismatch_ids = [a for a, _ in sample[20:25]]
|
||||
used_ids = [a for a, _ in random.sample([u for u in uploaded if u[0] not in set(missing_ids + mismatch_ids)], 200)]
|
||||
await bind_to_messages(s, token, chat, used_ids)
|
||||
bindings_before = await message_bindings(s, token, chat)
|
||||
assert {a for ids in bindings_before.values() for a in ids} == set(used_ids)
|
||||
orphans = ["att/%s/original" % uuid.uuid4() for _ in range(20)]
|
||||
mkeys, xkeys = keys_for(missing_ids), keys_for(mismatch_ids)
|
||||
code = ("import asyncio\nfrom app import config\nfrom app.s3 import S3\nasync def m():\n s=S3(); await s.start()\n"
|
||||
" for k in %r: await s.client.put_object(Bucket=config.S3_BUCKET, Key=k, Body=b'orphan')\n"
|
||||
" for k in %r: await s.delete(k)\n"
|
||||
" for k in %r: await s.client.put_object(Bucket=config.S3_BUCKET, Key=k, Body=b'x'*7)\n await s.stop()\nasyncio.run(m())" % (orphans, mkeys, xkeys))
|
||||
in_api(code)
|
||||
report["injected"] = {"orphan_objects": 20, "missing_objects": 20, "size_mismatch": 5}
|
||||
used_pairs = [u for u in uploaded if u[0] in set(used_ids)]
|
||||
time.sleep(6) # осиротевшие объекты станут старше grace=5 с
|
||||
# --- dry-run ---
|
||||
dry1 = reconcile("--grace", "5")
|
||||
dry2 = reconcile("--grace", "5")
|
||||
report["dry_run"] = {k: dry1[k] for k in ("orphan_objects_count", "missing_objects_count", "size_mismatch_count", "orphans_skipped_fresh", "timing_ms", "fixed")}
|
||||
report["dry_run_repeatable"] = all(dry1[k] == dry2[k] for k in ("orphan_objects_count", "missing_objects_count", "size_mismatch_count"))
|
||||
report["dry_run_found_exactly_injected"] = (dry1["orphan_objects_count"], dry1["missing_objects_count"], dry1["size_mismatch_count"]) == (20, 20, 5)
|
||||
# --- apply параллельно с загрузкой новых (свежих) вложений ---
|
||||
fresh_task = asyncio.create_task(upload_many(s, token, chat, 200, offset=100000))
|
||||
await asyncio.sleep(1.0)
|
||||
loop = asyncio.get_running_loop()
|
||||
applied = await loop.run_in_executor(None, lambda: reconcile("--grace", "5", "--apply"))
|
||||
fresh = await fresh_task
|
||||
report["apply"] = {k: applied[k] for k in ("fixed", "orphans_skipped_fresh", "timing_ms", "orphan_objects_count", "missing_objects_count", "size_mismatch_count")}
|
||||
report["fresh_uploads_during_apply"] = {"uploaded": len(fresh), "intact": await check_intact(s, token, fresh)}
|
||||
report["in_use_attachments_intact"] = "%d/%d" % (await check_intact(s, token, used_pairs), len(used_pairs))
|
||||
bindings_after = await message_bindings(s, token, chat)
|
||||
report["message_bindings"] = {"messages": len(bindings_before), "attachments": len(used_ids),
|
||||
"unchanged": bindings_after == bindings_before}
|
||||
again = reconcile("--grace", "5", "--apply")
|
||||
report["second_apply_idempotent"] = {"fixed": again["fixed"], "orphan_objects_count": again["orphan_objects_count"],
|
||||
"missing_objects_count": again["missing_objects_count"]}
|
||||
failed = sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c",
|
||||
"SELECT error_code || '=' || count(*) FROM attachments WHERE status = 'failed' AND chat_id = '%s' GROUP BY error_code ORDER BY 1" % chat).stdout.split()
|
||||
report["marked_failed"] = failed
|
||||
with open(os.path.join(args.out, "track2_reconcile.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
print(json.dumps({k: v for k, v in report.items() if k != "args"}, ensure_ascii=False, indent=2))
|
||||
assert len(uploaded) == 3 * (args.total // 3), "not all requested uploads succeeded"
|
||||
assert report["dry_run_repeatable"] and report["dry_run_found_exactly_injected"], report["dry_run"]
|
||||
assert dry1["fixed"] == dry2["fixed"] == {"deleted_orphans": 0, "marked_failed": 0}
|
||||
assert applied["fixed"] == {"deleted_orphans": 20, "marked_failed": 25}, applied
|
||||
assert report["fresh_uploads_during_apply"] == {"uploaded": 200, "intact": 200}
|
||||
assert report["in_use_attachments_intact"] == "200/200" and report["message_bindings"]["unchanged"]
|
||||
assert again["fixed"] == {"deleted_orphans": 0, "marked_failed": 0}, again
|
||||
assert (again["orphan_objects_count"], again["missing_objects_count"], again["size_mismatch_count"]) == (0, 0, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--total", type=int, default=3000)
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
@@ -0,0 +1,2 @@
|
||||
aiohttp==3.14.3
|
||||
Pillow==12.3.0
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: rolling update под нагрузкой. Нагрузка идёт через LB 60 с; на t=10 запускается scripts/rolling-update.sh.
|
||||
Проверяем: ошибки клиента (после повторов с тем же ключом), потерянные/задублированные сообщения, смену instance id."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import loadgen # noqa: E402
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
|
||||
|
||||
async def main(args):
|
||||
stand = loadgen.Stand("http://127.0.0.1:8080")
|
||||
await stand.setup()
|
||||
rec = loadgen.Recorder()
|
||||
info = {}
|
||||
|
||||
def update():
|
||||
time.sleep(max(0, rec.t0 + 10 - time.time()))
|
||||
info["start"] = round(time.time() - rec.t0, 2)
|
||||
out = subprocess.run(["sh", "scripts/rolling-update.sh", args.revision], cwd=ROOT, capture_output=True, text=True)
|
||||
info["end"] = round(time.time() - rec.t0, 2)
|
||||
info["script_exit"] = out.returncode
|
||||
info["script_output"] = out.stdout.strip().splitlines()[-6:]
|
||||
|
||||
threading.Thread(target=update, daemon=True).start()
|
||||
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
|
||||
await asyncio.sleep(1)
|
||||
check = await loadgen.verify(stand, rec)
|
||||
instances_before = {e[4] for e in rec.events if e[1] == "POST" and e[0] < info["start"] and e[4]}
|
||||
instances_during = {e[4] for e in rec.events if e[1] == "POST" and info["start"] <= e[0] <= info["end"] and e[4]}
|
||||
instances_after = {e[4] for e in rec.events if e[1] == "POST" and e[0] > info["end"] + 2 and e[4]}
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "update": info, "revision": args.revision,
|
||||
"instances_before": sorted(instances_before), "instances_during": sorted(instances_during),
|
||||
"instances_after": sorted(instances_after), "all_replicas_replaced": not (instances_before & instances_after),
|
||||
"before": loadgen.summarize(rec, 2, info["start"]), "during": loadgen.summarize(rec, info["start"], info["end"]),
|
||||
"after": loadgen.summarize(rec, info["end"] + 2, None), "total": loadgen.summarize(rec), "verification": check,
|
||||
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0 and check["final_failures"] == 0}
|
||||
with open(os.path.join(args.out, "rolling_update.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--seconds", type=int, default=60)
|
||||
ap.add_argument("--concurrency", type=int, default=16)
|
||||
ap.add_argument("--revision", default="rolling-%d" % int(time.time()))
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: ≥10 одновременных загрузок крупных файлов и пик памяти API.
|
||||
|
||||
Для каждого значения UPLOAD_CONCURRENCY (одновременных загрузок на реплику) пересоздаётся ОДНА реплика API
|
||||
(пик RSS обнуляется), затем 12 клиентов одновременно грузят ~8 MiB PNG. Пик памяти — VmHWM из /proc/1/status контейнера
|
||||
(максимум резидентной памяти за время жизни процесса). Результат: evidence/upload_memory.json.
|
||||
|
||||
python research/upload_memory.py --out evidence
|
||||
Требует Pillow и aiohttp (tools/.venv). Стенд должен быть запущен (docker compose up -d).
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import aiohttp
|
||||
from PIL import Image
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
PROJECT = "mtusi-lab4"
|
||||
|
||||
|
||||
def sh(*args, env=None):
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
env=dict(os.environ, **(env or {}))).stdout.strip()
|
||||
|
||||
|
||||
def noisy_png(side, seed):
|
||||
"""Почти несжимаемый PNG (шум): размер файла близок к side*side*3 байт."""
|
||||
raw = os.urandom(side * side * 3)
|
||||
img = Image.frombytes("RGB", (side, side), raw)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "PNG", compress_level=1)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def api_containers():
|
||||
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||||
"--filter", "label=com.docker.compose.service=api").split()
|
||||
|
||||
|
||||
def set_api(concurrency, replicas):
|
||||
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", "--scale", "api=%d" % replicas, "api",
|
||||
env={"UPLOAD_CONCURRENCY": str(concurrency)})
|
||||
deadline = time.time() + 90
|
||||
while time.time() < deadline:
|
||||
ids = api_containers()
|
||||
if len(ids) == replicas and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids):
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError("API replicas did not become healthy within 90 seconds")
|
||||
time.sleep(6)
|
||||
|
||||
|
||||
def peak_rss_mib(cid):
|
||||
out = sh("docker", "exec", cid, "sh", "-c", "grep -E 'VmHWM|VmRSS' /proc/1/status")
|
||||
vals = {l.split(":")[0]: int(l.split()[1]) / 1024 for l in out.splitlines()}
|
||||
return round(vals.get("VmHWM", 0), 1), round(vals.get("VmRSS", 0), 1)
|
||||
|
||||
|
||||
async def register(s):
|
||||
name = "up_%s" % uuid.uuid4().hex[:12]
|
||||
pw = "upload-test-pw-" + uuid.uuid4().hex[:6]
|
||||
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "u", "password": pw}) as r:
|
||||
user = await r.json()
|
||||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||||
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||||
sess = await r.json()
|
||||
return user["id"], sess["token"]
|
||||
|
||||
|
||||
async def one_run(clients, data, token, chat_id, timeout=120):
|
||||
results = []
|
||||
t0 = time.perf_counter()
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout), connector=aiohttp.TCPConnector(limit=clients)) as s:
|
||||
async def up(i):
|
||||
t = time.perf_counter()
|
||||
try:
|
||||
async with s.post("%s/api/v1/chats/%s/attachments?filename=load%d.png" % (BASE, chat_id, i), data=data,
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) as r:
|
||||
body = await r.json(content_type=None)
|
||||
results.append((r.status, time.perf_counter() - t, body.get("id") if r.status < 300 else body.get("error", {}).get("code")))
|
||||
except Exception as exc:
|
||||
results.append((type(exc).__name__, time.perf_counter() - t, None))
|
||||
await asyncio.gather(*(up(i) for i in range(clients)))
|
||||
return results, time.perf_counter() - t0
|
||||
|
||||
|
||||
async def main(args):
|
||||
data = noisy_png(args.side, 1)
|
||||
print("размер файла: %.1f MiB, sha256 %s" % (len(data) / 1048576, hashlib.sha256(data).hexdigest()[:12]))
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "file_mib": round(len(data) / 1048576, 2), "clients": args.clients,
|
||||
"api_memory_limit": "256m", "runs": []}
|
||||
for conc in args.concurrency:
|
||||
set_api(conc, 1)
|
||||
cid = api_containers()[0]
|
||||
limits = json.loads(sh("docker", "inspect", "-f",
|
||||
'{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', cid))
|
||||
assert limits == {"memory_bytes": 256 * 1024 * 1024, "nano_cpus": 1000000000}, limits
|
||||
async with aiohttp.ClientSession() as s:
|
||||
uid, token = await register(s)
|
||||
async with s.post(BASE + "/api/v1/chats", json={"title": "mem", "member_ids": [uid]}, headers={"Authorization": "Bearer " + token}) as r:
|
||||
chat = await r.json()
|
||||
baseline_hwm, _ = peak_rss_mib(cid)
|
||||
results, wall = await one_run(args.clients, data, token, chat["id"])
|
||||
peak, rss = peak_rss_mib(cid)
|
||||
statuses = {}
|
||||
for st, _, _ in results:
|
||||
statuses[str(st)] = statuses.get(str(st), 0) + 1
|
||||
ok_lat = sorted(t for st, t, _ in results if st == 201)
|
||||
# проверка целостности: скачиваем каждый принятый файл и сверяем sha256
|
||||
intact = 0
|
||||
async with aiohttp.ClientSession() as s:
|
||||
for st, _, att in results:
|
||||
if st == 201:
|
||||
async with s.get("%s/api/v1/attachments/%s/content" % (BASE, att), headers={"Authorization": "Bearer " + token}) as r:
|
||||
if hashlib.sha256(await r.read()).hexdigest() == hashlib.sha256(data).hexdigest():
|
||||
intact += 1
|
||||
oom = sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", cid)
|
||||
entry = {"UPLOAD_CONCURRENCY": conc, "statuses": statuses, "wall_seconds": round(wall, 2), "peak_rss_mib": peak,
|
||||
"resource_limits": limits,
|
||||
"rss_before_mib": baseline_hwm, "rss_after_mib": rss, "oom_killed": oom,
|
||||
"accepted_files_intact": "%d/%d" % (intact, statuses.get("201", 0)),
|
||||
"p50_s": round(ok_lat[len(ok_lat) // 2], 2) if ok_lat else None, "max_s": round(ok_lat[-1], 2) if ok_lat else None}
|
||||
print(json.dumps(entry, ensure_ascii=False))
|
||||
report["runs"].append(entry)
|
||||
set_api(4, 2)
|
||||
with open(os.path.join(args.out, "upload_memory.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
for entry in report["runs"]:
|
||||
assert entry["statuses"] == {"201": args.clients}, entry
|
||||
assert entry["accepted_files_intact"] == "%d/%d" % (args.clients, args.clients), entry
|
||||
assert entry["oom_killed"] == "false" and 0 < entry["peak_rss_mib"] < 256, entry
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--clients", type=int, default=12)
|
||||
ap.add_argument("--side", type=int, default=1700, help="сторона квадрата PNG-шума; 1700 -> около 8.7 MiB")
|
||||
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 12])
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Создаёт локальные секреты стенда (secrets/ в .gitignore). Идемпотентно: существующие файлы не перезаписываются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p secrets
|
||||
gen() { head -c "$1" /dev/urandom | base64 | tr -d '=+/\n'; }
|
||||
mk() { # mk <имя> <содержимое>
|
||||
if [ ! -s "secrets/$1" ]; then printf '%s' "$2" > "secrets/$1"; echo "создан secrets/$1"; fi
|
||||
}
|
||||
mk postgres_password "$(gen 24)"
|
||||
mk database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
|
||||
# S3 (SeaweedFS): администратор (только bootstrap) и отдельные ключи API с правами на ОДИН bucket
|
||||
mk s3_admin_access_key "adm$(gen 8)"
|
||||
mk s3_admin_secret_key "$(gen 30)"
|
||||
mk s3_access_key "api$(gen 8)"
|
||||
mk s3_secret_key "$(gen 30)"
|
||||
BUCKET="${S3_BUCKET:-messenger-attachments}"
|
||||
if [ ! -s secrets/seaweed_s3.json ]; then
|
||||
cat > secrets/seaweed_s3.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
{"name": "admin", "credentials": [{"accessKey": "$(cat secrets/s3_admin_access_key)", "secretKey": "$(cat secrets/s3_admin_secret_key)"}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "api", "credentials": [{"accessKey": "$(cat secrets/s3_access_key)", "secretKey": "$(cat secrets/s3_secret_key)"}],
|
||||
"actions": ["Read:$BUCKET", "Write:$BUCKET", "List:$BUCKET"]}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
echo "создан secrets/seaweed_s3.json"
|
||||
fi
|
||||
chmod 644 secrets/* 2>/dev/null || true # читает непривилегированный пользователь контейнера (uid 10001)
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
rem Windows analogue of restart.sh: restart the whole stand, volumes are kept; wait until healthy.
|
||||
cd /d "%~dp0.."
|
||||
docker compose --env-file .env restart postgres valkey seaweedfs api lb >NUL 2>&1
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs api lb >NUL 2>&1
|
||||
exit /b %errorlevel%
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Перезапуск ВСЕГО стенда с сохранением volumes: LB, API-реплики, PostgreSQL, Valkey, S3. Данные не удаляются.
|
||||
# После рестарта ждём healthy всех сервисов (`up --wait`), чтобы проверка сразу видела готовый стенд (S3 стартует ~10 с).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres valkey seaweedfs api lb >/dev/null 2>&1
|
||||
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs api lb >/dev/null 2>&1
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Возвращает стенд к заданному числу API-реплик (API_REPLICAS в .env, по умолчанию 2):
|
||||
# остановленные контейнеры удаляются и создаются заново; данные (volumes) не затрагиваются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
for cid in $(docker ps -aq --filter "label=com.docker.compose.project=mtusi-lab4" --filter "label=com.docker.compose.service=api" --filter "status=exited"); do
|
||||
docker rm "$cid" >/dev/null
|
||||
done
|
||||
N="${API_REPLICAS:-$(grep -E '^API_REPLICAS=' .env | cut -d= -f2 | tr -d '\r')}"
|
||||
docker compose --env-file .env up -d --no-deps --scale api="$N" api >/dev/null 2>&1 || { echo "не удалось вернуть $N реплик" >&2; exit 1; }
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# Rolling update API без простоя: реплики заменяются по одной.
|
||||
# 1) миграции выполняются ОДИН раз до замены (идемпотентно; должны быть обратно совместимы со старой версией);
|
||||
# 2) для каждой старой реплики: поднять новую (из нового образа) рядом -> дождаться healthy и попадания в LB ->
|
||||
# остановить старую штатно (SIGTERM -> drain 4 с -> graceful shutdown) -> удалить.
|
||||
# Использование: scripts/rolling-update.sh [REVISION]
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
REV="${1:-$(date +%s)}"
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
API_FILTER="--filter label=com.docker.compose.project=mtusi-lab4 --filter label=com.docker.compose.service=api"
|
||||
|
||||
$COMPOSE build --build-arg REV="$REV" api >/dev/null
|
||||
$COMPOSE run --rm migrate >/dev/null
|
||||
OLD=$(docker ps -q $API_FILTER)
|
||||
N=$(echo "$OLD" | wc -w | tr -d ' ')
|
||||
echo "revision=$REV replicas=$N"
|
||||
for old in $OLD; do
|
||||
BEFORE_FILE=$(mktemp)
|
||||
docker ps -q $API_FILTER | sort > "$BEFORE_FILE"
|
||||
$COMPOSE up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1
|
||||
NEW=$(docker ps -q $API_FILTER | sort | comm -13 "$BEFORE_FILE" -)
|
||||
rm -f "$BEFORE_FILE"
|
||||
for i in $(seq 1 60); do
|
||||
[ "$(docker inspect -f '{{.State.Health.Status}}' "$NEW" 2>/dev/null)" = healthy ] && break
|
||||
sleep 1
|
||||
done
|
||||
sleep 3 # LB: rise=2 проверки по 1 с -> реплика в ротации
|
||||
docker stop --time 15 "$old" >/dev/null
|
||||
docker rm "$old" >/dev/null
|
||||
echo "заменена реплика $old -> $NEW"
|
||||
done
|
||||
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
rem Windows analogue of stop-one.sh: stop exactly one API replica and leave it stopped.
|
||||
set "cid="
|
||||
echo %TARGET_INSTANCE_ID%| findstr /r "^[a-f0-9][a-f0-9]*$" >NUL || exit /b 2
|
||||
for /f %%i in ('docker ps -q --filter id^=%TARGET_INSTANCE_ID% --filter label^=com.docker.compose.project^=mtusi-lab4 --filter label^=com.docker.compose.service^=api') do set "cid=%%i"
|
||||
if "%cid%"=="" exit /b 3
|
||||
docker stop --time 0 %cid% >NUL 2>&1
|
||||
exit /b %errorlevel%
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Останавливает РОВНО одну API-реплику (TARGET_INSTANCE_ID == X-Instance-Id == короткий ID контейнера) и
|
||||
# оставляет её остановленной. Не трогает БД, Valkey, LB и volumes. Возвращается быстро (< 2 с).
|
||||
# `docker stop --time 0`: SIGTERM и немедленный SIGKILL — имитация отказа процесса; политика restart
|
||||
# после ручной остановки не поднимает контейнер. Восстановление: scripts/restore-replicas.sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
''|*[!a-f0-9]*) echo "некорректный TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
|
||||
--filter "label=com.docker.compose.project=mtusi-lab4" --filter "label=com.docker.compose.service=api")
|
||||
[ -n "$cid" ] || { echo "реплика не найдена среди работающих api-контейнеров" >&2; exit 3; }
|
||||
docker stop --time 0 "$cid" >/dev/null
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Вложения (lab4): потоковая загрузка PNG/JPEG в приватный S3, метаданные в PostgreSQL, авторизованное скачивание."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import uuid
|
||||
|
||||
from . import config, images
|
||||
from .web import ApiError, bad, format_dt, log, parse_uuid, unavailable
|
||||
|
||||
ALLOWED_TYPES = ("image/png", "image/jpeg")
|
||||
VARIANT_NAMES = ("original", "thumbnail", "crop", "optimized")
|
||||
ATT_COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, "
|
||||
"a.s3_key, a.created_at, a.updated_at")
|
||||
|
||||
|
||||
def attachment_json(row, variants=()):
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"],
|
||||
"content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
|
||||
"variants": list(variants), "error_code": row["error_code"], "created_at": format_dt(row["created_at"]),
|
||||
"updated_at": format_dt(row["updated_at"])}
|
||||
|
||||
|
||||
def valid_filename(query):
|
||||
values = query.get("filename")
|
||||
if not values or len(values) != 1:
|
||||
raise bad("Требуется параметр filename")
|
||||
name = values[0]
|
||||
if not 1 <= len(name) <= 255 or not name.strip():
|
||||
raise bad("filename: длина 1–255 символов")
|
||||
try:
|
||||
name.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise bad("filename содержит некорректные символы") from None
|
||||
if "/" in name or "\\" in name or "\x00" in name or any(unicodedata.category(ch) == "Cc" for ch in name):
|
||||
raise bad("filename не должен содержать / \\ NUL и управляющие символы")
|
||||
return name
|
||||
|
||||
|
||||
class StreamResponse:
|
||||
"""Потоковый ответ: тело читается из S3 порциями и сразу отдаётся клиенту (без загрузки файла в память)."""
|
||||
|
||||
def __init__(self, content_type, length, stream):
|
||||
self.content_type, self.length, self.stream = content_type, length, stream
|
||||
|
||||
async def send(self, send, request_id, instance_id):
|
||||
headers = [(b"content-type", self.content_type.encode()), (b"content-length", str(self.length).encode()),
|
||||
(b"x-request-id", request_id.encode()), (b"x-content-type-options", b"nosniff"),
|
||||
(b"cache-control", b"private, no-store")]
|
||||
if instance_id:
|
||||
headers.append((b"x-instance-id", instance_id.encode()))
|
||||
await send({"type": "http.response.start", "status": 200, "headers": headers})
|
||||
try:
|
||||
while True:
|
||||
chunk = await self.stream.read(config.DOWNLOAD_CHUNK_BYTES)
|
||||
if not chunk:
|
||||
break
|
||||
await send({"type": "http.response.body", "body": chunk, "more_body": True})
|
||||
finally:
|
||||
self.stream.close()
|
||||
await send({"type": "http.response.body", "body": b""})
|
||||
|
||||
|
||||
class AttachmentMixin:
|
||||
"""Подмешивается в Messenger: требует self.db, self.s3, self.authenticate()."""
|
||||
|
||||
def _init_attachments(self):
|
||||
self._upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
||||
self._upload_waiting = 0
|
||||
|
||||
async def _receive_upload(self, req):
|
||||
"""Принимает тело в SpooledTemporaryFile (до 1 MiB в памяти, далее диск), считает SHA-256, ограничивает размер."""
|
||||
declared = req.headers.get("content-length")
|
||||
too_large = ApiError(413, "payload_too_large", "Файл больше 10 MiB", {"connection": "close"})
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
raise bad("Некорректный Content-Length")
|
||||
if int(declared) > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
|
||||
digest, size = hashlib.sha256(), 0
|
||||
try:
|
||||
while True:
|
||||
event = await req.receive()
|
||||
if event["type"] == "http.disconnect":
|
||||
raise bad("Соединение закрыто клиентом")
|
||||
chunk = event.get("body", b"")
|
||||
size += len(chunk)
|
||||
if size > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
if not event.get("more_body"):
|
||||
break
|
||||
except BaseException:
|
||||
spool.close()
|
||||
raise
|
||||
if size == 0:
|
||||
spool.close()
|
||||
raise bad("Пустой файл")
|
||||
return spool, digest.hexdigest(), size
|
||||
|
||||
async def upload_attachment(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
chat_id = await self._require_member(user_id, params)
|
||||
filename = valid_filename(req.query)
|
||||
media = req.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media not in ALLOWED_TYPES:
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: image/png или image/jpeg")
|
||||
if self._upload_waiting >= config.UPLOAD_QUEUE:
|
||||
raise ApiError(429, "rate_limited", "Слишком много одновременных загрузок",
|
||||
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
self._upload_waiting += 1
|
||||
try:
|
||||
await self._upload_slots.acquire() # ограничение одновременных загрузок на реплику (RAM/диск/CPU)
|
||||
finally:
|
||||
self._upload_waiting -= 1
|
||||
try:
|
||||
spool, sha256, size = await self._receive_upload(req)
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
content_type, width, height = await loop.run_in_executor(self.image_pool, images.validate, spool, media)
|
||||
except images.ImageRejected as exc:
|
||||
raise ApiError(exc.status, exc.code, exc.message) from None
|
||||
att_id = uuid.uuid4()
|
||||
key = "att/%s/original" % att_id
|
||||
spool.seek(0)
|
||||
await self.s3.put(key, spool, content_type, size) # 1) байты в S3
|
||||
try: # 2) метаданные в PostgreSQL
|
||||
if "db_after_put" in config.FAULT_INJECTION: # только для экспериментов (по умолчанию пусто)
|
||||
raise RuntimeError("injected: сбой БД после записи в S3")
|
||||
row = await self.db.fetchrow(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, "
|
||||
"height, status, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'ready', $10) "
|
||||
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, "
|
||||
"s3_key, created_at, updated_at",
|
||||
att_id, uuid.UUID(chat_id), uuid.UUID(user_id), filename, content_type, size, sha256, width, height,
|
||||
key, retry=False)
|
||||
except BaseException:
|
||||
# компенсация: объект без метаданных удаляем сразу; если не вышло — его уберёт reconcile (grace period)
|
||||
try:
|
||||
if "s3_delete_fails" in config.FAULT_INJECTION:
|
||||
raise OSError("injected: компенсирующее удаление не удалось")
|
||||
await self.s3.delete(key)
|
||||
except Exception as exc:
|
||||
log("warning", "s3_orphan_left", error_type=type(exc).__name__)
|
||||
raise
|
||||
finally:
|
||||
spool.close()
|
||||
finally:
|
||||
self._upload_slots.release()
|
||||
return 201, attachment_json(row)
|
||||
|
||||
async def _attachment_for(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
att_id = parse_uuid(params["attachment_id"], "attachment_id")
|
||||
row = await self.db.fetchrow(
|
||||
"SELECT " + ATT_COLUMNS + " FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 "
|
||||
"WHERE a.id = $1", att_id, user_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
return row
|
||||
|
||||
async def get_attachment(self, req, params):
|
||||
row = await self._attachment_for(req, params)
|
||||
return 200, attachment_json(row)
|
||||
|
||||
async def download_attachment(self, req, params):
|
||||
row = await self._attachment_for(req, params)
|
||||
variant = "original"
|
||||
if "variant" in req.query:
|
||||
if len(req.query["variant"]) != 1 or req.query["variant"][0] not in VARIANT_NAMES:
|
||||
raise bad("variant: original | thumbnail | crop | optimized")
|
||||
variant = req.query["variant"][0]
|
||||
if row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение ещё не готово или не удалось обработать")
|
||||
if variant != "original":
|
||||
raise ApiError(404, "not_found", "Вариант отсутствует")
|
||||
stream, length = await self.s3.open_stream(row["s3_key"])
|
||||
return 200, StreamResponse(row["content_type"], length, stream)
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Конфигурация из переменных окружения (см. .env.example)."""
|
||||
import os
|
||||
|
||||
|
||||
def _int(name, default):
|
||||
value = os.environ.get(name)
|
||||
return int(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _float(name, default):
|
||||
value = os.environ.get(name)
|
||||
return float(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _secret(name, default=""):
|
||||
"""Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз."""
|
||||
if os.environ.get(name):
|
||||
return os.environ[name]
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path and os.path.exists(path):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return fh.read().strip()
|
||||
return default
|
||||
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = _int("APP_PORT", 8080)
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
|
||||
DATABASE_URL = _secret("DATABASE_URL")
|
||||
DB_POOL_MIN = _int("DB_POOL_MIN", 2)
|
||||
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
||||
DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула
|
||||
DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса
|
||||
DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0)
|
||||
|
||||
VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен
|
||||
VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2)
|
||||
VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0)
|
||||
|
||||
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60)
|
||||
|
||||
ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2)
|
||||
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP
|
||||
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
|
||||
HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4)
|
||||
|
||||
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
||||
MAX_INFLIGHT = _int("MAX_INFLIGHT", 64)
|
||||
LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000)
|
||||
RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1)
|
||||
MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations"))
|
||||
DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы
|
||||
GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8)
|
||||
KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5)
|
||||
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600)
|
||||
|
||||
# --- S3 / вложения (lab4) ---
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_ACCESS_KEY = _secret("S3_ACCESS_KEY")
|
||||
S3_SECRET_KEY = _secret("S3_SECRET_KEY")
|
||||
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2.0)
|
||||
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 15.0)
|
||||
S3_POOL = _int("S3_POOL", 20)
|
||||
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) # 10 MiB = 10 485 760
|
||||
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000)
|
||||
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 4) # одновременных загрузок на реплику
|
||||
UPLOAD_QUEUE = _int("UPLOAD_QUEUE", 16) # ожидающих загрузок; сверх — 429
|
||||
IMAGE_THREADS = _int("IMAGE_THREADS", 2)
|
||||
DOWNLOAD_CHUNK_BYTES = _int("DOWNLOAD_CHUNK_BYTES", 65536)
|
||||
ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) # объект без метаданных моложе этого возраста не трогаем
|
||||
FAULT_INJECTION = {x for x in os.environ.get("FAULT_INJECTION", "").split(",") if x} # db_after_put, s3_delete_fails (эксперименты)
|
||||
@@ -0,0 +1,71 @@
|
||||
"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503."""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .web import log, unavailable
|
||||
|
||||
DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError,
|
||||
asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError)
|
||||
_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError)
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.pool = None
|
||||
|
||||
async def start(self):
|
||||
if not config.DATABASE_URL:
|
||||
log("error", "database_url_missing")
|
||||
return
|
||||
# min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво.
|
||||
self.pool = await asyncpg.create_pool(
|
||||
dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX,
|
||||
command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT,
|
||||
max_inactive_connection_lifetime=60,
|
||||
server_settings={"application_name": "messenger-api", "timezone": "UTC"})
|
||||
|
||||
async def stop(self):
|
||||
if self.pool:
|
||||
try:
|
||||
await asyncio.wait_for(self.pool.close(), 3)
|
||||
except Exception:
|
||||
self.pool.terminate()
|
||||
|
||||
@asynccontextmanager
|
||||
async def conn(self):
|
||||
if self.pool is None:
|
||||
raise unavailable("База данных не настроена")
|
||||
try:
|
||||
async with self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT) as connection:
|
||||
yield connection
|
||||
except DEPENDENCY_ERRORS as exc:
|
||||
log("warning", "db_unavailable", error_type=type(exc).__name__)
|
||||
raise unavailable() from exc
|
||||
|
||||
async def _run(self, method, sql, args, retry):
|
||||
for attempt in (0, 1):
|
||||
try:
|
||||
async with self.conn() as c:
|
||||
return await getattr(c, method)(sql, *args)
|
||||
except Exception as exc:
|
||||
if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE):
|
||||
continue # разорванное соединение пула (рестарт БД): один повтор только для чтения
|
||||
raise
|
||||
|
||||
async def fetch(self, sql, *args):
|
||||
return await self._run("fetch", sql, args, True)
|
||||
|
||||
async def fetchrow(self, sql, *args, retry=True):
|
||||
return await self._run("fetchrow", sql, args, retry)
|
||||
|
||||
async def fetchval(self, sql, *args):
|
||||
return await self._run("fetchval", sql, args, True)
|
||||
|
||||
async def execute(self, sql, *args):
|
||||
return await self._run("execute", sql, args, False)
|
||||
|
||||
async def ping(self):
|
||||
return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0)
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Проверка загружаемых изображений ДО полного декодирования: сигнатура, формат Pillow, размеры, целостность.
|
||||
|
||||
PNG/JPEG определяются по содержимому (а не по Content-Type). Лимит пикселей проверяется по заголовку,
|
||||
поэтому «decompression bomb» отклоняется без декодирования. Целостность: PNG — verify() (CRC всех чанков, без
|
||||
распаковки в память), JPEG — декодирование в режиме draft (1/8 размера => около 1/64 памяти).
|
||||
"""
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from . import config
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None # лимит применяем сами по заголовку (ниже), без предупреждений Pillow
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
class ImageRejected(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
super().__init__(message)
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def sniff(head):
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return "image/png"
|
||||
if head[:3] == b"\xff\xd8\xff":
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def _check_png_stream(fileobj, width, height):
|
||||
"""Целостность потока IDAT без аллокации изображения: распаковка порциями, проверка конца потока и размера данных."""
|
||||
fileobj.seek(8)
|
||||
inflater = zlib.decompressobj()
|
||||
total, channels, bit_depth, interlaced = 0, 0, 0, 0
|
||||
while True:
|
||||
head = fileobj.read(8)
|
||||
if len(head) < 8:
|
||||
raise ValueError("truncated")
|
||||
length, kind = struct.unpack(">I4s", head)
|
||||
if length > 0x7FFFFFFF:
|
||||
raise ValueError("chunk")
|
||||
if kind == b"IHDR":
|
||||
data = fileobj.read(length)
|
||||
bit_depth, color_type, _, _, interlaced = struct.unpack(">BBBBB", data[8:13])
|
||||
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[color_type]
|
||||
fileobj.read(4)
|
||||
elif kind == b"IDAT":
|
||||
remaining = length
|
||||
while remaining:
|
||||
piece = fileobj.read(min(remaining, 65536))
|
||||
if not piece:
|
||||
raise ValueError("truncated")
|
||||
remaining -= len(piece)
|
||||
pending = piece
|
||||
while pending:
|
||||
total += len(inflater.decompress(pending, 1 << 20))
|
||||
pending = inflater.unconsumed_tail
|
||||
fileobj.read(4)
|
||||
elif kind == b"IEND":
|
||||
break
|
||||
else:
|
||||
fileobj.seek(length + 4, 1)
|
||||
if not inflater.eof:
|
||||
raise ValueError("zlib stream incomplete")
|
||||
if not interlaced:
|
||||
row = (width * channels * bit_depth + 7) // 8
|
||||
if total != height * (row + 1):
|
||||
raise ValueError("unexpected image data size")
|
||||
|
||||
|
||||
def validate(fileobj, declared_type):
|
||||
"""Возвращает (content_type, width, height) либо бросает ImageRejected (415 — не то/повреждено, 413 — много пикселей)."""
|
||||
fileobj.seek(0)
|
||||
detected = sniff(fileobj.read(16))
|
||||
if detected is None or detected != declared_type:
|
||||
raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG/JPEG или заявленному Content-Type")
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
with Image.open(fileobj) as img:
|
||||
width, height = img.size
|
||||
fmt = img.format
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
|
||||
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено или не поддерживается") from None
|
||||
if fmt not in ("PNG", "JPEG") or width < 1 or height < 1:
|
||||
raise ImageRejected(415, "unsupported_media_type", "Неподдерживаемый формат изображения")
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise ImageRejected(413, "payload_too_large", "Слишком много пикселей в изображении")
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
img = Image.open(fileobj)
|
||||
if fmt == "PNG":
|
||||
img.verify()
|
||||
_check_png_stream(fileobj, width, height)
|
||||
else:
|
||||
img.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
img.load()
|
||||
img.close()
|
||||
except Exception: # любая ошибка декодера = повреждённый файл
|
||||
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from None
|
||||
return detected, width, height
|
||||
+409
@@ -0,0 +1,409 @@
|
||||
"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey."""
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config, passwords
|
||||
from .attachments import AttachmentMixin, StreamResponse
|
||||
from .db import DEPENDENCY_ERRORS, Database
|
||||
from .s3 import S3
|
||||
from .sessions import SessionCache, Sessions
|
||||
from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params,
|
||||
parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE)
|
||||
|
||||
ROUTES = [
|
||||
("health_live", re.compile(r"/health/live"), {"GET"}),
|
||||
("health_ready", re.compile(r"/health/ready"), {"GET"}),
|
||||
("users", re.compile(r"/api/v1/users"), {"POST"}),
|
||||
("users_me", re.compile(r"/api/v1/users/me"), {"GET"}),
|
||||
("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}),
|
||||
("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}),
|
||||
("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}),
|
||||
("chat", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)"), {"GET"}),
|
||||
("messages", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/messages"), {"GET", "POST"}),
|
||||
("upload_attachment", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/attachments"), {"POST"}),
|
||||
("get_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)"), {"GET"}),
|
||||
("download_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)/content"), {"GET"}),
|
||||
]
|
||||
ROUTE_LABELS = {
|
||||
"health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users",
|
||||
"users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions",
|
||||
"session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats",
|
||||
"chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages",
|
||||
"upload_attachment": "/api/v1/chats/{chat_id}/attachments", "get_attachment": "/api/v1/attachments/{attachment_id}",
|
||||
"download_attachment": "/api/v1/attachments/{attachment_id}/content",
|
||||
}
|
||||
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
BEARER = 'Bearer realm="messenger"'
|
||||
BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps.
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12]
|
||||
|
||||
MSG_ATTS = ("COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma "
|
||||
"WHERE ma.message_id = messages.id), '{}'::uuid[]) AS attachment_ids")
|
||||
CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, "
|
||||
"(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ")
|
||||
|
||||
|
||||
def user_json(row):
|
||||
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
|
||||
"created_at": format_dt(row["created_at"])}
|
||||
|
||||
|
||||
def chat_json(row):
|
||||
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]],
|
||||
"created_at": format_dt(row["created_at"])}
|
||||
|
||||
|
||||
def message_json(row):
|
||||
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
|
||||
"text": row["text"], "created_at": format_dt(row["created_at"]),
|
||||
"attachment_ids": [str(x) for x in (row["attachment_ids"] or [])]}
|
||||
|
||||
|
||||
def unauthorized(challenge=BEARER, message="Требуется действительная сессия"):
|
||||
return ApiError(401, "unauthorized", message, {"www-authenticate": challenge})
|
||||
|
||||
|
||||
class Messenger(AttachmentMixin):
|
||||
def __init__(self):
|
||||
self.db = Database()
|
||||
self.s3 = S3()
|
||||
self.image_pool = ThreadPoolExecutor(max_workers=config.IMAGE_THREADS, thread_name_prefix="image")
|
||||
self._init_attachments()
|
||||
self.cache = SessionCache()
|
||||
self.sessions = Sessions(self.db, self.cache)
|
||||
self.inflight = 0
|
||||
self.draining = False
|
||||
|
||||
# --- аутентификация ------------------------------------------------------
|
||||
@staticmethod
|
||||
def bearer_token(req):
|
||||
header = req.headers.get("authorization", "")
|
||||
scheme, _, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()):
|
||||
raise unauthorized()
|
||||
return token.strip()
|
||||
|
||||
async def authenticate(self, req):
|
||||
token = self.bearer_token(req)
|
||||
user_id = await self.sessions.resolve(token)
|
||||
if user_id is None:
|
||||
raise unauthorized()
|
||||
req.user_id, req.token = user_id, token
|
||||
return user_id
|
||||
|
||||
# --- обработчики -----------------------------------------------------------
|
||||
async def health_live(self, req, params):
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
async def health_ready(self, req, params):
|
||||
# Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API.
|
||||
if self.draining:
|
||||
raise unavailable("Сервис останавливается")
|
||||
try:
|
||||
await self.db.ping()
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception:
|
||||
raise unavailable() from None
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
async def users(self, req, params):
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"username", "display_name", "password"},
|
||||
{"username", "display_name", "password"})
|
||||
username = data["username"]
|
||||
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
|
||||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
display_name = valid_text(data["display_name"], "display_name", 100)
|
||||
password = valid_text(data["password"], "password", 128, allow_empty=True)
|
||||
if len(password) < 12:
|
||||
raise bad("password должен содержать от 12 до 128 символов")
|
||||
pw_hash = await passwords.hash_password(password)
|
||||
row = await self.db.fetchrow(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) "
|
||||
"ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at",
|
||||
uuid.uuid4(), username, display_name, pw_hash)
|
||||
if row is None:
|
||||
raise ApiError(409, "conflict", "username уже занят")
|
||||
return 201, user_json(row)
|
||||
|
||||
async def users_me(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id)
|
||||
if row is None:
|
||||
raise unauthorized()
|
||||
return 200, user_json(row)
|
||||
|
||||
async def create_session(self, req, params):
|
||||
header = req.headers.get("authorization", "")
|
||||
scheme, _, value = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized(BASIC, "Неверные учётные данные")
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
username, sep, password = decoded.partition(":")
|
||||
if not sep:
|
||||
raise ValueError
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized(BASIC, "Неверные учётные данные") from None
|
||||
row = None
|
||||
if USERNAME_RE.fullmatch(username):
|
||||
row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users "
|
||||
"WHERE username = $1", username)
|
||||
ok = await passwords.verify_password(row["password_hash"] if row else None, password)
|
||||
if not ok:
|
||||
raise unauthorized(BASIC, "Неверные учётные данные")
|
||||
token, expires_at = await self.sessions.create(row["id"])
|
||||
return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)}
|
||||
|
||||
async def session_current(self, req, params):
|
||||
await self.authenticate(req)
|
||||
await self.sessions.revoke(req.token)
|
||||
return 204, None
|
||||
|
||||
async def chats(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
if req.method == "GET":
|
||||
limit, after = page_params(req.query, "c", user_id)
|
||||
if after is None:
|
||||
rows = await self.db.fetch(
|
||||
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
|
||||
"ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1)
|
||||
else:
|
||||
rows = await self.db.fetch(
|
||||
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
|
||||
"AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4",
|
||||
user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
|
||||
return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor}
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"})
|
||||
title = valid_text(data["title"], "title", 100)
|
||||
members = data["member_ids"]
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
raise bad("member_ids должен содержать от 1 до 100 элементов")
|
||||
ids = [parse_uuid(m, "member_id") for m in members]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise bad("member_ids содержит дубликаты")
|
||||
if user_id not in ids:
|
||||
raise bad("Создатель должен быть в member_ids")
|
||||
chat_id = uuid.uuid4()
|
||||
uuids = [uuid.UUID(i) for i in ids]
|
||||
try:
|
||||
async with self.db.conn() as c:
|
||||
async with c.transaction(): # чат и участники создаются атомарно
|
||||
# Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном
|
||||
# участнике вставка падает и откатывается вся транзакция, включая уже созданный чат.
|
||||
created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at",
|
||||
chat_id, title)
|
||||
await c.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) "
|
||||
"SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)",
|
||||
chat_id, created, uuids)
|
||||
except asyncpg.ForeignKeyViolationError:
|
||||
raise bad("Неизвестный участник") from None
|
||||
return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)}
|
||||
|
||||
async def _require_member(self, user_id, params):
|
||||
chat_id = parse_uuid(params["chat_id"], "chat_id")
|
||||
ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id)
|
||||
if not ok:
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
return chat_id
|
||||
|
||||
async def chat(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
chat_id = parse_uuid(params["chat_id"], "chat_id")
|
||||
row = await self.db.fetchrow(
|
||||
CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1",
|
||||
chat_id, user_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
return 200, chat_json(row)
|
||||
|
||||
async def messages(self, req, params):
|
||||
user_id = await self.authenticate(req)
|
||||
if req.method == "POST":
|
||||
row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id)
|
||||
return 201, message_json(row)
|
||||
chat_id = await self._require_member(user_id, params)
|
||||
if req.method == "GET":
|
||||
limit, after = page_params(req.query, "m", chat_id)
|
||||
if after is None:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
|
||||
"ORDER BY created_at, id LIMIT $2", chat_id, limit + 1)
|
||||
else:
|
||||
rows = await self.db.fetch(
|
||||
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
|
||||
"AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4",
|
||||
chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
|
||||
return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor}
|
||||
|
||||
async def _post_message(self, req, chat_id, user_id):
|
||||
# Сначала разбор запроса; ошибка валидации (400/415) возвращается только участнику чата — постороннему всегда 404.
|
||||
try:
|
||||
key = req.headers.get("idempotency-key", "")
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
|
||||
raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})")
|
||||
req.require_json_media_type()
|
||||
data = parse_json_object(await req.body(), {"text"}, {"text", "attachment_ids"})
|
||||
raw_atts = data.get("attachment_ids", [])
|
||||
if not isinstance(raw_atts, list) or len(raw_atts) > 10:
|
||||
raise bad("attachment_ids: массив до 10 элементов")
|
||||
atts = [parse_uuid(a, "attachment_id") for a in raw_atts]
|
||||
if len(set(atts)) != len(atts):
|
||||
raise bad("attachment_ids содержит дубликаты")
|
||||
text = valid_text(data["text"], "text", 4000, allow_empty=bool(atts))
|
||||
except ApiError:
|
||||
await self._require_member(user_id, {"chat_id": chat_id})
|
||||
raise
|
||||
# Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == [].
|
||||
request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": atts}, sort_keys=True,
|
||||
ensure_ascii=False).encode("utf-8")).digest()
|
||||
row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6, $7)", uuid.UUID(chat_id),
|
||||
uuid.UUID(user_id), key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS),
|
||||
[uuid.UUID(a) for a in atts], retry=False)
|
||||
if row["outcome"] == "not_member":
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
if row["outcome"] == "attachment_not_found":
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
if row["outcome"] == "attachment_failed":
|
||||
raise ApiError(409, "conflict", "Вложение не удалось обработать")
|
||||
if bytes(row["request_hash"]) != request_hash:
|
||||
raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса")
|
||||
return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"],
|
||||
"created_at": row["msg_created"], "attachment_ids": row["msg_atts"]}
|
||||
|
||||
# --- ASGI ----------------------------------------------------------------------
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] == "lifespan":
|
||||
return await self.lifespan(receive, send)
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
started = time.perf_counter()
|
||||
request_id = new_request_id()
|
||||
route_name, status, headers = "unknown", 500, {}
|
||||
try:
|
||||
req = Request(scope, receive, request_id)
|
||||
handler, params, route_name = self.route(req.method, req.path)
|
||||
if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT:
|
||||
raise ApiError(429, "rate_limited", "Слишком много одновременных запросов",
|
||||
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
self.inflight += 1
|
||||
try:
|
||||
await asyncio.sleep(0)
|
||||
status, body = await handler(req, params)
|
||||
finally:
|
||||
self.inflight -= 1
|
||||
except ApiError as exc:
|
||||
status, headers = exc.status, exc.headers
|
||||
body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}
|
||||
except DEPENDENCY_ERRORS as exc:
|
||||
log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__)
|
||||
status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)}
|
||||
body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}}
|
||||
except Exception as exc:
|
||||
log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__)
|
||||
status = 500
|
||||
body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}
|
||||
if isinstance(body, StreamResponse):
|
||||
try:
|
||||
await body.send(send, request_id, INSTANCE_ID)
|
||||
except Exception as exc: # клиент оборвал загрузку/ошибка S3 посреди потока
|
||||
log("warning", "stream_aborted", request_id=request_id, error_type=type(exc).__name__)
|
||||
else:
|
||||
await respond(send, status, body, request_id, headers, INSTANCE_ID)
|
||||
if os.environ.get("ACCESS_LOG", "1") != "0":
|
||||
log("info", "request", request_id=request_id, method=scope.get("method"),
|
||||
route=ROUTE_LABELS.get(route_name, "unmatched"), status=status,
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 3))
|
||||
|
||||
async def janitor(self):
|
||||
"""Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике)."""
|
||||
import random
|
||||
await asyncio.sleep(random.uniform(5, 30))
|
||||
while True:
|
||||
try:
|
||||
await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys "
|
||||
"WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)",
|
||||
float(config.IDEMPOTENCY_TTL_SECONDS) + 3600)
|
||||
await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions "
|
||||
"WHERE expires_at < now() - interval '1 day' LIMIT 5000)")
|
||||
except Exception as exc:
|
||||
log("warning", "janitor_failed", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
def route(self, method, path):
|
||||
for name, pattern, methods in ROUTES:
|
||||
m = pattern.fullmatch(path)
|
||||
if m:
|
||||
if method not in methods:
|
||||
raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))})
|
||||
return getattr(self, name), m.groupdict(), name
|
||||
raise ApiError(404, "not_found", "Ресурс не найден")
|
||||
|
||||
async def lifespan(self, receive, send):
|
||||
while True:
|
||||
event = await receive()
|
||||
if event["type"] == "lifespan.startup":
|
||||
await self.db.start()
|
||||
await self.cache.start()
|
||||
await self.s3.start()
|
||||
self._janitor = asyncio.create_task(self.janitor())
|
||||
log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL),
|
||||
session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX)
|
||||
await send({"type": "lifespan.startup.complete"})
|
||||
elif event["type"] == "lifespan.shutdown":
|
||||
self.draining = True
|
||||
self._janitor.cancel()
|
||||
await self.cache.stop()
|
||||
await self.s3.stop()
|
||||
await self.db.stop()
|
||||
log("info", "shutdown")
|
||||
await send({"type": "lifespan.shutdown.complete"})
|
||||
return
|
||||
|
||||
|
||||
async def overloaded_app(scope, receive, send):
|
||||
"""Ответ uvicorn при превышении --limit-concurrency в контрактном формате."""
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
request_id = new_request_id()
|
||||
body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений",
|
||||
"request_id": request_id}}
|
||||
await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID)
|
||||
|
||||
|
||||
def _install_overload_response():
|
||||
for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"):
|
||||
try:
|
||||
module = __import__(name, fromlist=["service_unavailable"])
|
||||
except ImportError:
|
||||
continue
|
||||
module.service_unavailable = overloaded_app
|
||||
|
||||
|
||||
def create_app():
|
||||
_install_overload_response()
|
||||
return Messenger()
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Применение версионируемых SQL-миграций: python -m app.migrate
|
||||
|
||||
Свойства:
|
||||
* повторный запуск ничего не делает (таблица schema_migrations);
|
||||
* одновременный запуск нескольких экземпляров безопасен (advisory lock);
|
||||
* каждая миграция — одна транзакция вместе с записью о применении;
|
||||
* ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
LOCK_ID = 7352001
|
||||
|
||||
|
||||
async def connect(deadline):
|
||||
while True:
|
||||
try:
|
||||
return await asyncpg.connect(config.DATABASE_URL, timeout=3)
|
||||
except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log("info", "waiting_for_database", error_type=type(exc).__name__)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def main():
|
||||
wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60"))
|
||||
conn = await connect(time.monotonic() + wait)
|
||||
try:
|
||||
await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID)
|
||||
await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())")
|
||||
applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")}
|
||||
directory = os.path.abspath(config.MIGRATIONS_DIR)
|
||||
files = sorted(f for f in os.listdir(directory) if f.endswith(".sql"))
|
||||
count = 0
|
||||
for name in files:
|
||||
sql = open(os.path.join(directory, name), encoding="utf-8").read()
|
||||
checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest()
|
||||
if name in applied:
|
||||
if applied[name] != checksum:
|
||||
log("error", "migration_modified", version=name)
|
||||
sys.exit(2)
|
||||
continue
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum)
|
||||
count += 1
|
||||
log("info", "migration_applied", version=name)
|
||||
log("info", "migrations_done", applied=count, total=len(files))
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop."""
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
|
||||
from . import config
|
||||
|
||||
_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB,
|
||||
parallelism=config.ARGON2_PARALLELISM)
|
||||
_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2")
|
||||
# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя.
|
||||
_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation")
|
||||
|
||||
|
||||
async def hash_password(password):
|
||||
return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password)
|
||||
|
||||
|
||||
def _verify(stored, password):
|
||||
try:
|
||||
return _hasher.verify(stored, password)
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
async def verify_password(stored, password):
|
||||
"""stored=None -> проверка фиктивного хеша, результат всегда False."""
|
||||
ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password)
|
||||
return ok and stored is not None
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Сверка S3 и PostgreSQL (reconcile): python -m app.reconcile [--apply] [--grace СЕКУНДЫ] [--prefix att/]
|
||||
|
||||
Находит и (с --apply) безопасно исправляет расхождения двух хранилищ:
|
||||
* orphan_object — объект в S3 без строки в attachments (сбой между записью в S3 и COMMIT метаданных, прерванная загрузка);
|
||||
удаляется ТОЛЬКО если старше grace (по умолчанию ORPHAN_GRACE_SECONDS=900 с): свежий объект может быть
|
||||
загрузкой, метаданные которой ещё не зафиксированы;
|
||||
* missing_object — строка attachments (ready) без объекта в S3 (потеря/удаление объекта): строка НЕ удаляется,
|
||||
статус -> failed, error_code=object_missing (сообщения со ссылкой остаются, скачивание даёт 409);
|
||||
* size_mismatch — размер объекта не совпадает с size_bytes: статус -> failed, error_code=size_mismatch.
|
||||
Режим по умолчанию — dry-run (ничего не меняет). Повторный запуск идемпотентен. Используемые (привязанные к сообщению)
|
||||
вложения и свежие объекты не удаляются никогда: удаляются только объекты, у которых нет строки в БД.
|
||||
Выход: JSON-отчёт со счётчиками и временем обхода.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
from .s3 import S3
|
||||
|
||||
|
||||
async def reconcile(apply, grace, prefix):
|
||||
started = time.perf_counter()
|
||||
s3 = S3()
|
||||
await s3.start()
|
||||
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
|
||||
try:
|
||||
t0 = time.perf_counter()
|
||||
rows = await conn.fetch("SELECT id, s3_key, size_bytes, status, created_at FROM attachments")
|
||||
db_ms = (time.perf_counter() - t0) * 1000
|
||||
by_key = {r["s3_key"]: r for r in rows}
|
||||
t0 = time.perf_counter()
|
||||
objects = {}
|
||||
async for key, size, modified in s3.list_keys(prefix):
|
||||
objects[key] = (size, modified)
|
||||
list_ms = (time.perf_counter() - t0) * 1000
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=grace)
|
||||
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
|
||||
"orphan_objects": [], "orphans_skipped_fresh": 0, "missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
|
||||
for key, (size, modified) in objects.items():
|
||||
row = by_key.get(key)
|
||||
if row is None:
|
||||
if modified < cutoff:
|
||||
report["orphan_objects"].append(key)
|
||||
else:
|
||||
report["orphans_skipped_fresh"] += 1
|
||||
elif row["status"] == "ready" and row["size_bytes"] != size:
|
||||
report["size_mismatch"].append(str(row["id"]))
|
||||
for key, row in by_key.items():
|
||||
if key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
|
||||
report["missing_objects"].append(str(row["id"]))
|
||||
if apply:
|
||||
for key in report["orphan_objects"]:
|
||||
# повторная проверка перед удалением: за время обхода строка могла появиться
|
||||
if await conn.fetchval("SELECT 1 FROM attachments WHERE s3_key = $1", key) is None:
|
||||
await s3.delete(key)
|
||||
report["fixed"]["deleted_orphans"] += 1
|
||||
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
|
||||
for att_id in ids:
|
||||
n = await conn.execute("UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() "
|
||||
"WHERE id = $1 AND status = 'ready'", att_id, code)
|
||||
report["fixed"]["marked_failed"] += int(n.split()[-1])
|
||||
report["timing_ms"] = {"db_query": round(db_ms, 1), "s3_listing": round(list_ms, 1),
|
||||
"total": round((time.perf_counter() - started) * 1000, 1)}
|
||||
return report
|
||||
finally:
|
||||
await conn.close()
|
||||
await s3.stop()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--apply", action="store_true", help="исправлять (по умолчанию dry-run)")
|
||||
ap.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
|
||||
ap.add_argument("--prefix", default="att/")
|
||||
ap.add_argument("--max-listed", type=int, default=20, help="сколько идентификаторов печатать в отчёте")
|
||||
args = ap.parse_args()
|
||||
report = asyncio.run(reconcile(args.apply, args.grace, args.prefix))
|
||||
for k in ("orphan_objects", "missing_objects", "size_mismatch"):
|
||||
report[k + "_count"] = len(report[k])
|
||||
report[k] = report[k][:args.max_listed]
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Приватное S3-хранилище (MinIO) через aiobotocore: потоковая загрузка/скачивание, таймауты, перевод сбоев в 503."""
|
||||
import asyncio
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from . import config
|
||||
from .web import log, unavailable
|
||||
|
||||
S3_ERRORS = (BotoCoreError, ClientError, asyncio.TimeoutError, OSError)
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self):
|
||||
self._ctx = None
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
if not config.S3_ENDPOINT:
|
||||
log("error", "s3_endpoint_missing")
|
||||
return
|
||||
session = get_session()
|
||||
self._ctx = session.create_client(
|
||||
"s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY, aws_secret_access_key=config.S3_SECRET_KEY,
|
||||
config=AioConfig(connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT,
|
||||
retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=config.S3_POOL,
|
||||
s3={"addressing_style": "path"}))
|
||||
self.client = await self._ctx.__aenter__()
|
||||
|
||||
async def stop(self):
|
||||
if self._ctx:
|
||||
await self._ctx.__aexit__(None, None, None)
|
||||
|
||||
def _check(self):
|
||||
if self.client is None:
|
||||
raise unavailable("Хранилище файлов не настроено")
|
||||
|
||||
async def put(self, key, fileobj, content_type, size):
|
||||
self._check()
|
||||
try:
|
||||
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentType=content_type,
|
||||
ContentLength=size)
|
||||
except S3_ERRORS as exc:
|
||||
log("warning", "s3_put_failed", error_type=type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
|
||||
async def open_stream(self, key):
|
||||
"""Возвращает (stream, content_length). Вызывающий обязан закрыть stream."""
|
||||
self._check()
|
||||
try:
|
||||
resp = await self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except S3_ERRORS as exc:
|
||||
log("warning", "s3_get_failed", error_type=type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
return resp["Body"], resp["ContentLength"]
|
||||
|
||||
async def delete(self, key):
|
||||
self._check()
|
||||
await self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
|
||||
async def head(self, key):
|
||||
"""Метаданные объекта или None, если его нет."""
|
||||
self._check()
|
||||
try:
|
||||
return await self.client.head_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
|
||||
return None
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
except S3_ERRORS as exc:
|
||||
raise unavailable("Хранилище файлов недоступно") from exc
|
||||
|
||||
async def ping(self):
|
||||
self._check()
|
||||
await asyncio.wait_for(self.client.head_bucket(Bucket=config.S3_BUCKET), 2.0)
|
||||
|
||||
async def list_keys(self, prefix):
|
||||
"""Асинхронный итератор (key, size, last_modified) по префиксу — для сверки (reconcile)."""
|
||||
self._check()
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
async for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
yield obj["Key"], obj["Size"], obj["LastModified"]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Идемпотентный bootstrap S3: python -m app.s3_bootstrap (one-shot сервис s3-init при каждом `docker compose up`).
|
||||
|
||||
1) bucket создаётся администратором, если его ещё нет (повтор ничего не меняет);
|
||||
2) проверяется, что анонимный запрос к bucket отклоняется (private);
|
||||
3) проверяется, что отдельный ключ API (права только на этот bucket) может писать/читать/удалять и НЕ может создавать bucket.
|
||||
Ключи читаются из файлов секретов; в вывод не попадают. Код выхода != 0, если любая проверка не пройдена.
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
def secret(name):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return fh.read().strip()
|
||||
|
||||
|
||||
def client(session, access, secret_key):
|
||||
return session.create_client("s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=AioConfig(connect_timeout=3, read_timeout=10, retries={"max_attempts": 1},
|
||||
s3={"addressing_style": "path"}))
|
||||
|
||||
|
||||
async def main():
|
||||
bucket = config.S3_BUCKET
|
||||
session = get_session()
|
||||
deadline = time.time() + 60
|
||||
async with client(session, secret("S3_ADMIN_ACCESS_KEY"), secret("S3_ADMIN_SECRET_KEY")) as admin:
|
||||
while True:
|
||||
try:
|
||||
await admin.list_buckets()
|
||||
break
|
||||
except Exception as exc:
|
||||
if time.time() > deadline:
|
||||
print("S3 недоступен:", type(exc).__name__)
|
||||
return 1
|
||||
await asyncio.sleep(1)
|
||||
try:
|
||||
await admin.head_bucket(Bucket=bucket)
|
||||
print("bucket %s уже существует" % bucket)
|
||||
except ClientError:
|
||||
await admin.create_bucket(Bucket=bucket)
|
||||
print("bucket %s создан" % bucket)
|
||||
# приватность: запрос без подписи отклоняется
|
||||
try:
|
||||
urllib.request.urlopen("%s/%s/" % (config.S3_ENDPOINT, bucket), timeout=5)
|
||||
print("ОШИБКА: bucket доступен анонимно")
|
||||
return 2
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code not in (401, 403):
|
||||
print("ОШИБКА: анонимный доступ вернул %d, ожидался 401/403" % exc.code)
|
||||
return 2
|
||||
print("анонимный доступ запрещён (HTTP %d)" % exc.code)
|
||||
# ключ API: рабочие права на bucket, нет прав администратора
|
||||
async with client(session, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) as api:
|
||||
key = "bootstrap-check/%d" % int(time.time())
|
||||
await api.put_object(Bucket=bucket, Key=key, Body=b"ok")
|
||||
got = await (await api.get_object(Bucket=bucket, Key=key))["Body"].read()
|
||||
await api.delete_object(Bucket=bucket, Key=key)
|
||||
if got != b"ok":
|
||||
print("ОШИБКА: чтение/запись ключом API")
|
||||
return 3
|
||||
try:
|
||||
await api.create_bucket(Bucket="not-allowed-" + str(int(time.time())))
|
||||
print("ОШИБКА: ключ API смог создать bucket (избыточные права)")
|
||||
return 4
|
||||
except ClientError as exc:
|
||||
print("ключ API ограничен одним bucket (create_bucket отклонён: %s)" % exc.response["Error"]["Code"])
|
||||
print("bootstrap завершён")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Точка входа: uvicorn с управляемым завершением (drain).
|
||||
|
||||
SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик
|
||||
исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown).
|
||||
Так плановая остановка/rolling update не даёт ошибок клиентам.
|
||||
"""
|
||||
import signal
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from . import config
|
||||
from .main import app
|
||||
from .web import log
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
_draining = False
|
||||
|
||||
def handle_exit(self, sig, frame):
|
||||
if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0:
|
||||
self._draining = True
|
||||
app.draining = True
|
||||
log("info", "draining", seconds=config.DRAIN_SECONDS)
|
||||
timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
return
|
||||
super().handle_exit(sig, frame)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False,
|
||||
log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS,
|
||||
timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY)
|
||||
DrainingServer(cfg).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through).
|
||||
|
||||
Политики:
|
||||
* токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена;
|
||||
* expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS);
|
||||
* инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey;
|
||||
* Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным;
|
||||
circuit breaker не даёт каждому запросу ждать таймаут;
|
||||
* окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS.
|
||||
"""
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
|
||||
from . import config
|
||||
from .web import log
|
||||
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
except ImportError: # кэш опционален
|
||||
aioredis = None
|
||||
|
||||
|
||||
def token_hash(token):
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self):
|
||||
self.client = None
|
||||
self.down_until = 0.0
|
||||
self.stats = {"hit": 0, "miss": 0, "error": 0}
|
||||
|
||||
async def start(self):
|
||||
if config.VALKEY_URL and aioredis:
|
||||
self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT,
|
||||
socket_connect_timeout=config.VALKEY_TIMEOUT)
|
||||
|
||||
async def stop(self):
|
||||
if self.client:
|
||||
await self.client.aclose()
|
||||
|
||||
def _usable(self):
|
||||
return self.client is not None and time.monotonic() >= self.down_until
|
||||
|
||||
def _fail(self, exc):
|
||||
self.stats["error"] += 1
|
||||
if time.monotonic() >= self.down_until:
|
||||
log("warning", "session_cache_unavailable", error_type=type(exc).__name__)
|
||||
self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS
|
||||
|
||||
async def get(self, key):
|
||||
if not self._usable():
|
||||
return None
|
||||
try:
|
||||
value = await self.client.get(b"s:" + key)
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
return None
|
||||
self.stats["hit" if value is not None else "miss"] += 1
|
||||
return value
|
||||
|
||||
async def put(self, key, value, ttl):
|
||||
if not self._usable() or ttl <= 0:
|
||||
return
|
||||
try:
|
||||
await self.client.set(b"s:" + key, value, ex=max(1, int(ttl)))
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
|
||||
async def delete(self, key):
|
||||
if not self._usable():
|
||||
return
|
||||
try:
|
||||
await self.client.delete(b"s:" + key)
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
|
||||
async def ping(self):
|
||||
if self.client is None:
|
||||
return None
|
||||
try:
|
||||
return bool(await self.client.ping())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class Sessions:
|
||||
def __init__(self, db, cache):
|
||||
self.db, self.cache = db, cache
|
||||
|
||||
@staticmethod
|
||||
def _cache_value(user_id, expires_at):
|
||||
return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode()
|
||||
|
||||
async def create(self, user_id):
|
||||
token = secrets.token_urlsafe(32) # 256 бит энтропии
|
||||
digest = token_hash(token)
|
||||
row = await self.db.fetchrow(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) "
|
||||
"VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at",
|
||||
digest, user_id, float(config.SESSION_TTL_SECONDS))
|
||||
remaining = row["expires_at"].timestamp() - time.time()
|
||||
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
|
||||
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
|
||||
return token, row["expires_at"]
|
||||
|
||||
async def resolve(self, token):
|
||||
"""token -> user_id (str) либо None, если сессии нет/истекла/отозвана."""
|
||||
digest = token_hash(token)
|
||||
cached = await self.cache.get(digest)
|
||||
if cached is not None:
|
||||
user_id, _, expires_ms = cached.decode().partition("|")
|
||||
if int(expires_ms) > time.time() * 1000:
|
||||
return user_id
|
||||
await self.cache.delete(digest)
|
||||
row = await self.db.fetchrow(
|
||||
"SELECT user_id, expires_at FROM sessions "
|
||||
"WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest)
|
||||
if row is None:
|
||||
return None
|
||||
user_id = str(row["user_id"])
|
||||
remaining = row["expires_at"].timestamp() - time.time()
|
||||
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
|
||||
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
|
||||
return user_id
|
||||
|
||||
async def revoke(self, token):
|
||||
digest = token_hash(token)
|
||||
await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest)
|
||||
await self.cache.delete(digest)
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал."""
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from . import config
|
||||
|
||||
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
LIMIT_RE = re.compile(r"[0-9]{1,9}")
|
||||
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def bad(message):
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unavailable(message="Зависимость временно недоступна"):
|
||||
return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)})
|
||||
|
||||
|
||||
def log(level, msg, **fields):
|
||||
record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "msg": msg}
|
||||
record.update(fields)
|
||||
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
# --- время -------------------------------------------------------------------
|
||||
def format_dt(value):
|
||||
value = value.astimezone(timezone.utc)
|
||||
return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond
|
||||
|
||||
|
||||
def dt_to_us(value):
|
||||
return (value - EPOCH) // timedelta(microseconds=1)
|
||||
|
||||
|
||||
def us_to_dt(us):
|
||||
return EPOCH + timedelta(microseconds=us)
|
||||
|
||||
|
||||
# --- валидация ---------------------------------------------------------------
|
||||
def parse_uuid(value, what="id"):
|
||||
if not isinstance(value, str) or not UUID_RE.fullmatch(value):
|
||||
raise bad("Некорректный %s" % what)
|
||||
return value.lower()
|
||||
|
||||
|
||||
def valid_text(value, field, maximum, allow_empty=False):
|
||||
if not isinstance(value, str):
|
||||
raise bad("Поле %s должно быть строкой" % field)
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise bad("Поле %s содержит некорректные символы" % field) from None
|
||||
if "\x00" in value:
|
||||
raise bad("Поле %s содержит NUL" % field)
|
||||
if len(value) > maximum:
|
||||
raise bad("Поле %s слишком длинное" % field)
|
||||
if not allow_empty and not value.strip():
|
||||
raise bad("Поле %s не может быть пустым" % field)
|
||||
return value
|
||||
|
||||
|
||||
def _reject_constant(name):
|
||||
raise ValueError(name)
|
||||
|
||||
|
||||
def parse_json_object(raw, required, allowed):
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant)
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad("Тело запроса — некорректный JSON") from None
|
||||
if not isinstance(data, dict):
|
||||
raise bad("Ожидается JSON-объект")
|
||||
if set(data) - allowed:
|
||||
raise bad("Лишние поля в запросе")
|
||||
if any(key not in data for key in required):
|
||||
raise bad("Не хватает обязательных полей")
|
||||
return data
|
||||
|
||||
|
||||
def encode_cursor(kind, resource, key):
|
||||
payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":"))
|
||||
return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(text, kind, resource):
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode())
|
||||
if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource
|
||||
or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool)
|
||||
or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"])
|
||||
or not -62135596800000000 < data["t"] < 253402300799000000):
|
||||
raise ValueError
|
||||
return data["t"], data["i"]
|
||||
except (ValueError, UnicodeError):
|
||||
raise bad("Некорректный cursor") from None
|
||||
|
||||
|
||||
def page_params(query, kind, resource):
|
||||
limit, cursor = 50, None
|
||||
if "limit" in query:
|
||||
if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]):
|
||||
raise bad("limit должен быть целым числом 1–100")
|
||||
limit = int(query["limit"][0])
|
||||
if not 1 <= limit <= 100:
|
||||
raise bad("limit должен быть целым числом 1–100")
|
||||
if "cursor" in query:
|
||||
if len(query["cursor"]) != 1 or not query["cursor"][0]:
|
||||
raise bad("Некорректный cursor")
|
||||
cursor = decode_cursor(query["cursor"][0], kind, resource)
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def parse_query(raw):
|
||||
result = {}
|
||||
if raw:
|
||||
for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True):
|
||||
result.setdefault(key, []).append(value)
|
||||
return result
|
||||
|
||||
|
||||
class Request:
|
||||
def __init__(self, scope, receive, request_id):
|
||||
self.method = scope["method"]
|
||||
self.path = scope["path"]
|
||||
self.query = parse_query(scope.get("query_string", b""))
|
||||
self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
|
||||
self.receive = receive
|
||||
self.request_id = request_id
|
||||
self.user_id = None
|
||||
|
||||
async def body(self):
|
||||
declared = self.headers.get("content-length")
|
||||
if declared is not None:
|
||||
if not declared.isdigit():
|
||||
raise bad("Некорректный Content-Length")
|
||||
if int(declared) > config.MAX_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
|
||||
chunks, size = [], 0
|
||||
while True:
|
||||
event = await self.receive()
|
||||
if event["type"] == "http.disconnect":
|
||||
raise bad("Соединение закрыто клиентом")
|
||||
chunk = event.get("body", b"")
|
||||
size += len(chunk)
|
||||
if size > config.MAX_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
|
||||
chunks.append(chunk)
|
||||
if not event.get("more_body"):
|
||||
return b"".join(chunks)
|
||||
|
||||
def require_json_media_type(self):
|
||||
media = self.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json")
|
||||
|
||||
|
||||
async def respond(send, status, body, request_id, extra=None, instance_id=None):
|
||||
payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
headers = [(b"x-request-id", request_id.encode())]
|
||||
if status != 204:
|
||||
headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())]
|
||||
else:
|
||||
headers.append((b"content-length", b"0"))
|
||||
if instance_id:
|
||||
headers.append((b"x-instance-id", instance_id.encode()))
|
||||
headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()]
|
||||
await send({"type": "http.response.start", "status": status, "headers": headers})
|
||||
await send({"type": "http.response.body", "body": payload})
|
||||
|
||||
|
||||
def new_request_id():
|
||||
return uuid.uuid4().hex
|
||||
@@ -0,0 +1,44 @@
|
||||
# Собственные проверки ЛР4
|
||||
|
||||
Сначала запустите стенд по [REPORT.md](../REPORT.md). Все команды выполняются из корня репозитория. В Windows задайте `$env:PYTHONUTF8='1'`.
|
||||
|
||||
```sh
|
||||
python -m unittest discover -s student-tests -p 'test_api.py' -v
|
||||
python -m unittest discover -s student-tests -p 'test_idempotency.py' -v
|
||||
python -m unittest discover -s student-tests -p 'test_attachments.py' -v
|
||||
python -m unittest discover -s student-tests -p 'test_topology.py' -v
|
||||
```
|
||||
|
||||
Это 38 тестов: API и аутентификация (12), идемпотентность между репликами (9), вложения и права доступа (11), топология (6).
|
||||
|
||||
Следующие 8 тестов временно останавливают зависимости или пересоздают API с настройками отказа/коротким TTL. Запускайте их последовательно на учебном стенде; данные и volumes сохраняются, настройки восстанавливаются после проверки.
|
||||
|
||||
```sh
|
||||
python student-tests/test_s3_failures.py -v
|
||||
python student-tests/test_expiry.py -v
|
||||
python student-tests/test_dependencies.py -v
|
||||
```
|
||||
|
||||
В проверке reconciler файл действительно прикреплён к сообщению: после очистки проверяются и файл, и ссылка сообщения. Свежий orphan проверяется с `--apply --grace 900`: он должен остаться; с `--grace 0` после завершения загрузок он удаляется.
|
||||
|
||||
Повторяемость bootstrap и ограниченные права ключа API:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env run --rm -T s3-init
|
||||
docker compose --env-file .env run --rm -T s3-init
|
||||
```
|
||||
|
||||
Публичные проверки запускаются отдельно, без изменений: `make check`, `make validate`, `make test`, сценарии `tests/resilience.py` из отчёта. На Windows можно запустить соответствующие Python-команды из Makefile. После failover восстановите вторую реплику: `docker compose --env-file .env start api`.
|
||||
|
||||
## Эксперименты
|
||||
|
||||
Для исследовательских скриптов нужны `aiohttp` и `Pillow`: `python -m pip install -r research/requirements.txt`. Создайте каталог результата заранее. Эксперименты запускайте последовательно; `upload_memory.py` временно меняет число реплик и лимит параллельных загрузок, затем возвращает две реплики и лимит 4.
|
||||
|
||||
```sh
|
||||
python research/upload_memory.py --clients 12 --out evidence
|
||||
python research/reconcile_bench.py --total 3000 --out evidence
|
||||
```
|
||||
|
||||
`reconcile_bench.py` создаёт синтетические данные, внедряет 20 orphan, 20 missing и 5 size mismatch, сравнивает dry-run/apply/повтор, проверяет SHA-256 и сохранность 200 вложений в 20 сообщениях. Начальный запуск `--apply --grace 0` убирает следы прежних экспериментов; скрипт предназначен для отдельного учебного стенда без посторонних загрузок.
|
||||
|
||||
Реальный повторный прогон: [checks_2026-11-11.txt](../evidence/checks_2026-11-11.txt), [результаты исследований](../evidence/recheck_2026-11-11/).
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 109 KiB |
@@ -0,0 +1,208 @@
|
||||
"""Собственные тесты lab2 против запущенного стенда (docker compose up -d).
|
||||
|
||||
python -m unittest discover -s student-tests -v # BASE_URL по умолчанию http://127.0.0.1:8080
|
||||
Тесты используют только синтетические данные и ничего не удаляют. Проверки expiry и недоступности
|
||||
зависимостей лежат отдельно (test_expiry.py, test_dependencies.py): они управляют контейнерами.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
ROOT = os.path.join(os.path.dirname(__file__), "..")
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, headers=None, raw=None):
|
||||
h = dict(headers or {})
|
||||
if method == "POST" and path.endswith("/messages") and headers is None:
|
||||
h["Idempotency-Key"] = uuid.uuid4().hex # lab3: ключ обязателен; тесты без ключа — в test_idempotency.py
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
h.setdefault("Content-Type", "application/json")
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
status, hdrs, payload = resp.status, resp.headers, resp.read()
|
||||
except urllib.error.HTTPError as err:
|
||||
status, hdrs, payload = err.code, err.headers, err.read()
|
||||
return status, hdrs, (json.loads(payload) if payload else None)
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()}
|
||||
|
||||
|
||||
def register(prefix="u"):
|
||||
name = "%s_%s" % (prefix, uuid.uuid4().hex[:12])
|
||||
password = "Pa55:word-" + uuid.uuid4().hex # двоеточие в пароле допустимо
|
||||
status, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "Имя " + name, "password": password})
|
||||
assert status == 201, user
|
||||
status, _, session = call("POST", "/api/v1/auth/sessions", headers=basic(name, password))
|
||||
assert status == 201, session
|
||||
return {"user": user, "password": password, "token": session["token"], "session": session}
|
||||
|
||||
|
||||
def psql(sql):
|
||||
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger",
|
||||
"-d", "messenger", "-Atc", sql], cwd=ROOT, capture_output=True, text=True, check=True)
|
||||
return out.stdout.strip()
|
||||
|
||||
|
||||
class AuthTest(unittest.TestCase):
|
||||
def assertError(self, result, status, code):
|
||||
self.assertEqual(result[0], status, result[2])
|
||||
self.assertEqual(result[2]["error"]["code"], code)
|
||||
self.assertEqual(result[2]["error"]["request_id"], result[1]["X-Request-Id"])
|
||||
|
||||
def test_register_validation(self):
|
||||
base = {"username": "val_" + uuid.uuid4().hex[:8], "display_name": "x", "password": "p" * 12}
|
||||
for patch in ({"password": "short"}, {"password": "p" * 129}, {"password": 12345678901234}, {"extra": 1},
|
||||
{"username": "BAD"}, {"display_name": " "}):
|
||||
body = dict(base, **patch)
|
||||
self.assertError(call("POST", "/api/v1/users", body), 400, "invalid_request")
|
||||
missing = {k: v for k, v in base.items() if k != "password"}
|
||||
self.assertError(call("POST", "/api/v1/users", missing), 400, "invalid_request")
|
||||
self.assertEqual(call("POST", "/api/v1/users", dict(base, password="я" * 12))[0], 201) # 12 code points
|
||||
|
||||
def test_password_is_not_exposed_or_stored_plain(self):
|
||||
actor = register()
|
||||
self.assertEqual(set(actor["user"]), {"id", "username", "display_name", "created_at"})
|
||||
stored = psql("SELECT password_hash FROM users WHERE username = '%s'" % actor["user"]["username"])
|
||||
self.assertTrue(stored.startswith("$argon2id$"), stored[:12])
|
||||
self.assertNotIn(actor["password"], stored)
|
||||
|
||||
def test_wrong_password_and_unknown_user_are_indistinguishable(self):
|
||||
actor = register()
|
||||
wrong = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], "wrong-password-123"))
|
||||
unknown = call("POST", "/api/v1/auth/sessions", headers=basic("nobody_" + uuid.uuid4().hex[:8], "wrong-password-123"))
|
||||
for r in (wrong, unknown):
|
||||
self.assertError(r, 401, "unauthorized")
|
||||
self.assertEqual(r[1]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"')
|
||||
self.assertEqual(wrong[2]["error"]["message"], unknown[2]["error"]["message"])
|
||||
for bad in ({}, {"Authorization": "Basic !!!"}, {"Authorization": "Basic " + base64.b64encode(b"nocolon").decode()},
|
||||
{"Authorization": "Bearer abc"}):
|
||||
self.assertEqual(call("POST", "/api/v1/auth/sessions", headers=bad)[0], 401)
|
||||
|
||||
def test_bearer_rules(self):
|
||||
actor = register()
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[2], actor["user"])
|
||||
for headers in ({}, basic(actor["user"]["username"], actor["password"]), {"X-User-Id": actor["user"]["id"]},
|
||||
{"Authorization": "Bearer " + "A" * 43}, {"Authorization": "Bearer short"}):
|
||||
r = call("GET", "/api/v1/users/me", headers=headers)
|
||||
self.assertError(r, 401, "unauthorized")
|
||||
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
|
||||
other = register()
|
||||
me = call("GET", "/api/v1/users/me", token=actor["token"], headers={"X-User-Id": other["user"]["id"]})[2]
|
||||
self.assertEqual(me["id"], actor["user"]["id"]) # X-User-Id игнорируется
|
||||
|
||||
def test_session_shape_entropy_and_logout(self):
|
||||
actor = register()
|
||||
s = actor["session"]
|
||||
self.assertEqual(s["token_type"], "Bearer")
|
||||
self.assertGreaterEqual(len(s["token"]), 43) # 32 байта в base64url = 256 бит
|
||||
self.assertNotIn(actor["user"]["id"], s["token"])
|
||||
self.assertNotEqual(register()["token"], s["token"])
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"])[0], 204)
|
||||
self.assertError(call("GET", "/api/v1/users/me", token=s["token"]), 401, "unauthorized")
|
||||
self.assertError(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"]), 401, "unauthorized")
|
||||
|
||||
def test_logout_revokes_in_database_and_cache(self):
|
||||
actor = register()
|
||||
call("GET", "/api/v1/users/me", token=actor["token"]) # прогреваем кэш
|
||||
call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])
|
||||
revoked = psql("SELECT count(*) FROM sessions s JOIN users u ON u.id = s.user_id "
|
||||
"WHERE u.username = '%s' AND s.revoked_at IS NOT NULL" % actor["user"]["username"])
|
||||
self.assertEqual(revoked, "1")
|
||||
|
||||
def test_token_only_stored_as_hash(self):
|
||||
actor = register()
|
||||
count = psql("SELECT count(*) FROM sessions WHERE token_hash = '\\x00'::bytea OR encode(token_hash,'escape') = '%s'" % actor["token"])
|
||||
self.assertEqual(count, "0")
|
||||
|
||||
def test_concurrent_registration_of_same_username(self):
|
||||
name = "race_" + uuid.uuid4().hex[:10]
|
||||
body = {"username": name, "display_name": "x", "password": "racing-password-1"}
|
||||
with ThreadPoolExecutor(12) as pool:
|
||||
codes = sorted(pool.map(lambda _: call("POST", "/api/v1/users", body)[0], range(12)))
|
||||
self.assertEqual(codes, [201] + [409] * 11)
|
||||
self.assertEqual(psql("SELECT count(*) FROM users WHERE username = '%s'" % name), "1")
|
||||
|
||||
|
||||
class DataTest(unittest.TestCase):
|
||||
def test_chat_creation_is_atomic(self):
|
||||
a = register()
|
||||
before = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
|
||||
ghost = str(uuid.uuid4())
|
||||
r = call("POST", "/api/v1/chats", {"title": "откат", "member_ids": [a["user"]["id"], ghost]}, token=a["token"])
|
||||
self.assertEqual(r[0], 400)
|
||||
after = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
|
||||
self.assertEqual(before, after, "после неудачного создания не должно остаться чата или участников")
|
||||
ok = call("POST", "/api/v1/chats", {"title": "ок", "member_ids": [a["user"]["id"]]}, token=a["token"])
|
||||
self.assertEqual(ok[0], 201)
|
||||
|
||||
def test_foreign_chat_empty_message_bad_json(self):
|
||||
a, b, eve = register(), register(), register()
|
||||
chat = call("POST", "/api/v1/chats", {"title": "ЛС", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
self.assertEqual(call("POST", path, {"text": "чужое"}, eve["token"])[0], 404)
|
||||
self.assertEqual(call("GET", path, token=eve["token"])[0], 404)
|
||||
self.assertEqual(call("POST", path, {"text": " "}, a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, raw=b"{oops", token=a["token"], headers={"Content-Type": "application/json"})[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "sender_id": b["user"]["id"]}, a["token"])[0], 400)
|
||||
sent = call("POST", path, {"text": "Привет 👋"}, a["token"])[2]
|
||||
self.assertEqual(sent["sender_id"], a["user"]["id"])
|
||||
|
||||
def test_multi_page_history_and_cursor_binding(self):
|
||||
a, b = register(), register()
|
||||
chat = call("POST", "/api/v1/chats", {"title": "стр", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
|
||||
other = call("POST", "/api/v1/chats", {"title": "др", "member_ids": [a["user"]["id"]]}, a["token"])[2]
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
sent = [call("POST", path, {"text": "m%d" % i}, (a if i % 2 else b)["token"])[2] for i in range(23)]
|
||||
walked, cursor, pages = [], None, 0
|
||||
while True:
|
||||
q = "?limit=%d" % (5 if pages % 2 == 0 else 8) + ("&cursor=" + quote(cursor, safe="") if cursor else "")
|
||||
page = call("GET", path + q, token=b["token"])[2]
|
||||
walked += page["items"]
|
||||
pages += 1
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
self.assertEqual(walked, sent)
|
||||
self.assertGreaterEqual(pages, 3)
|
||||
first = call("GET", path + "?limit=1", token=a["token"])[2]["next_cursor"]
|
||||
r = call("GET", "/api/v1/chats/%s/messages?cursor=%s" % (other["id"], quote(first, safe="")), token=a["token"])
|
||||
self.assertEqual(r[0], 400)
|
||||
|
||||
def test_parallel_writes_from_many_clients(self):
|
||||
users = [register() for _ in range(5)]
|
||||
chat = call("POST", "/api/v1/chats", {"title": "гонки", "member_ids": [u["user"]["id"] for u in users]}, users[0]["token"])[2]
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
|
||||
def client(u):
|
||||
return [call("POST", path, {"text": "%s-%d" % (u["user"]["username"], i)}, u["token"])[0] for i in range(20)]
|
||||
|
||||
with ThreadPoolExecutor(5) as pool:
|
||||
codes = [c for part in pool.map(client, users) for c in part]
|
||||
self.assertEqual(codes, [201] * 100)
|
||||
total, cursor = [], None
|
||||
while True:
|
||||
page = call("GET", path + "?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), token=users[0]["token"])[2]
|
||||
total += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
self.assertEqual(len({m["id"] for m in total}), 100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Собственные тесты вложений lab4 (негативные сценарии и права). Запуск:
|
||||
python -m unittest discover -s student-tests -p "test_attachments.py" -v (BASE_URL по умолчанию http://127.0.0.1:8080)
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import BASE, ROOT, call, psql, register # noqa: E402
|
||||
|
||||
FIX = os.path.join(os.path.dirname(__file__), "fixtures")
|
||||
PNG = open(os.path.join(FIX, "sample.png"), "rb").read()
|
||||
JPG = open(os.path.join(FIX, "sample.jpg"), "rb").read()
|
||||
|
||||
|
||||
def upload(token, chat_id, data, ctype="image/png", filename="sample.png", raw_filename=None):
|
||||
qs = raw_filename if raw_filename is not None else "filename=" + filename
|
||||
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?%s" % (BASE, chat_id, qs), data=data, method="POST",
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.headers, json.loads(r.read())
|
||||
except urllib.error.HTTPError as err:
|
||||
body = err.read()
|
||||
return err.code, err.headers, (json.loads(body) if body else None)
|
||||
|
||||
|
||||
def download(token, att_id, suffix="/content"):
|
||||
req = urllib.request.Request("%s/api/v1/attachments/%s%s" % (BASE, att_id, suffix),
|
||||
headers={"Authorization": "Bearer " + token if token else ""})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.headers, r.read()
|
||||
except urllib.error.HTTPError as err:
|
||||
return err.code, err.headers, err.read()
|
||||
|
||||
|
||||
def png_chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def bomb_png(width=20000, height=20000):
|
||||
"""Корректный по структуре PNG, заявляющий 400 млн пикселей, без данных изображения (отклоняется по заголовку)."""
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", ihdr) + png_chunk(b"IDAT", zlib.compress(b"\x00" * 16)) + png_chunk(b"IEND", b"")
|
||||
|
||||
|
||||
class AttachmentTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.a, self.b, self.eve = register("att"), register("att"), register("att")
|
||||
self.chat = call("POST", "/api/v1/chats", {"title": "файлы", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2]
|
||||
self.other = call("POST", "/api/v1/chats", {"title": "другой", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
|
||||
|
||||
def assertError(self, result, status, code=None):
|
||||
self.assertEqual(result[0], status, result[2])
|
||||
if code:
|
||||
self.assertEqual(result[2]["error"]["code"], code)
|
||||
|
||||
def test_png_and_jpeg_roundtrip_with_checksum(self):
|
||||
for data, ctype, name in ((PNG, "image/png", "фото.png"), (JPG, "image/jpeg", "photo.jpg")):
|
||||
status, headers, att = upload(self.a["token"], self.chat["id"], data, ctype, urllib.parse.quote(name))
|
||||
self.assertEqual(status, 201, att)
|
||||
self.assertEqual((att["status"], att["variants"], att["error_code"]), ("ready", [], None))
|
||||
self.assertEqual((att["size_bytes"], att["sha256"], att["content_type"]), (len(data), hashlib.sha256(data).hexdigest(), ctype))
|
||||
self.assertEqual((att["chat_id"], att["owner_id"], att["filename"]), (self.chat["id"], self.a["user"]["id"], name))
|
||||
code, hdr, body = download(self.b["token"], att["id"]) # другой участник чата
|
||||
self.assertEqual((code, body, hdr.get_content_type()), (200, data, ctype))
|
||||
self.assertEqual(download(self.b["token"], att["id"], "")[0], 200)
|
||||
|
||||
def test_too_large_is_413_and_nothing_is_stored(self):
|
||||
before = psql("SELECT count(*) FROM attachments")
|
||||
r = upload(self.a["token"], self.chat["id"], PNG[:8] + b"\x00" * (10 * 1024 * 1024 + 1))
|
||||
self.assertError(r, 413, "payload_too_large")
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
|
||||
|
||||
def test_corrupted_and_spoofed_files_are_415_not_500(self):
|
||||
cases = {
|
||||
"truncated png": (PNG[: len(PNG) // 2], "image/png"),
|
||||
"garbage png": (b"\x89PNG\r\n\x1a\n" + b"\x00" * 200, "image/png"),
|
||||
"random bytes": (os.urandom(500), "image/png"),
|
||||
"truncated jpeg": (JPG[: len(JPG) // 2], "image/jpeg"),
|
||||
"garbage jpeg": (b"\xff\xd8\xff\xe0" + b"\x00" * 200, "image/jpeg"),
|
||||
"png as jpeg": (PNG, "image/jpeg"),
|
||||
"jpeg as png": (JPG, "image/png"),
|
||||
"text as png": (b"hello, i am not an image", "image/png"),
|
||||
"gif": (b"GIF89a" + b"\x00" * 50, "image/png"),
|
||||
}
|
||||
for name, (data, ctype) in cases.items():
|
||||
with self.subTest(name):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], data, ctype), 415, "unsupported_media_type")
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "text/plain"), 415)
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "multipart/form-data; boundary=x"), 415)
|
||||
|
||||
def test_decompression_bomb_header_is_413(self):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], bomb_png()), 413, "payload_too_large")
|
||||
# 6000x6600 = 39.6 млн пикселей < лимита: по размерам допустимо, но данных изображения нет -> файл повреждён (415)
|
||||
self.assertEqual(upload(self.a["token"], self.chat["id"], bomb_png(6000, 6600))[0], 415)
|
||||
|
||||
def test_dangerous_filenames_are_400(self):
|
||||
bad = ["a%2Fb.png", "..%5Cx.png", "x%00.png", "x%0A.png", "x%7F.png", "", "%20%20", "n" * 256]
|
||||
for name in bad:
|
||||
with self.subTest(name):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + name), 400, "invalid_request")
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename=""), 400)
|
||||
self.assertEqual(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + "n" * 255)[0], 201)
|
||||
|
||||
def test_empty_body_is_400(self):
|
||||
self.assertError(upload(self.a["token"], self.chat["id"], b""), 400)
|
||||
|
||||
def test_foreign_user_cannot_upload_read_or_download(self):
|
||||
self.assertError(upload(self.eve["token"], self.chat["id"], PNG), 404)
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
for suffix in ("", "/content"):
|
||||
self.assertEqual(download(self.eve["token"], att["id"], suffix)[0], 404)
|
||||
self.assertEqual(download(None, att["id"], suffix)[0], 401)
|
||||
self.assertEqual(download(self.a["token"], str(uuid.uuid4()))[0], 404)
|
||||
self.assertEqual(download(self.a["token"], "not-a-uuid")[0], 400)
|
||||
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=thumbnail")[0], 404) # вариантов в lab4 нет
|
||||
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=bogus")[0], 400)
|
||||
|
||||
def test_message_with_attachments_and_chat_binding(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
foreign_chat_att = upload(self.a["token"], self.other["id"], JPG, "image/jpeg")[2]
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
|
||||
self.assertEqual(sent[0], 201, sent[2])
|
||||
self.assertEqual(sent[2]["attachment_ids"], [att["id"]])
|
||||
self.assertEqual(call("POST", path, {"text": "ещё раз то же вложение", "attachment_ids": [att["id"]]}, self.b["token"])[0], 201)
|
||||
listed = call("GET", path + "?limit=10", token=self.b["token"])[2]["items"]
|
||||
self.assertEqual([m["attachment_ids"] for m in listed], [[att["id"]], [att["id"]]])
|
||||
# вложение другого чата / несуществующее -> 404; дубликаты, >10, пустое сообщение без вложений -> 400
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [foreign_chat_att["id"]]}, self.a["token"])[0], 404)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, self.a["token"])[0], 404)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"], att["id"]]}, self.a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, self.a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": " ", "attachment_ids": []}, self.a["token"])[0], 400)
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": "nope"}, self.a["token"])[0], 400)
|
||||
# чужой пользователь не может сослаться на вложение
|
||||
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"]]}, self.eve["token"])[0], 404)
|
||||
|
||||
def test_idempotent_replay_with_attachments(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
key = uuid.uuid4().hex
|
||||
h = {"Idempotency-Key": key}
|
||||
first = call("POST", path, {"text": "t", "attachment_ids": [att["id"]]}, self.a["token"], headers=h)
|
||||
again = call("POST", path, {"attachment_ids": [att["id"]], "text": "t"}, self.a["token"], headers=h) # другой порядок ключей
|
||||
self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2]))
|
||||
self.assertEqual(call("POST", path, {"text": "t"}, self.a["token"], headers=h)[0], 409) # другое тело
|
||||
text_key = uuid.uuid4().hex
|
||||
m1 = call("POST", path, {"text": "без вложений"}, self.a["token"], headers={"Idempotency-Key": text_key})
|
||||
m2 = call("POST", path, {"text": "без вложений", "attachment_ids": []}, self.a["token"], headers={"Idempotency-Key": text_key})
|
||||
self.assertEqual(m1[2], m2[2]) # отсутствие == []
|
||||
self.assertEqual(m1[2]["attachment_ids"], [])
|
||||
|
||||
def test_aborted_upload_leaves_nothing(self):
|
||||
before = (psql("SELECT count(*) FROM attachments"))
|
||||
host, port = BASE.split("//")[1].split(":")
|
||||
with socket.create_connection((host, int(port))) as s:
|
||||
s.sendall(("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n"
|
||||
"Content-Type: image/png\r\nContent-Length: 200000\r\n\r\n" % (self.chat["id"], self.a["token"])).encode() + PNG[:5000])
|
||||
import time
|
||||
time.sleep(1)
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
|
||||
|
||||
def test_bucket_is_private_outside_the_api(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"])
|
||||
import subprocess
|
||||
code = ("import urllib.request,urllib.error,sys\n"
|
||||
"try:\n urllib.request.urlopen('http://seaweedfs:8333/messenger-attachments/%s', timeout=5); print('OPEN')\n"
|
||||
"except urllib.error.HTTPError as e: print(e.code)\n" % key)
|
||||
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT,
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
self.assertEqual(out, "403", "анонимный GET объекта напрямую из S3 должен быть запрещён")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины).
|
||||
|
||||
python student-tests/test_dependencies.py
|
||||
Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания:
|
||||
* Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает;
|
||||
* PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ),
|
||||
/health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, basic, call, register # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True)
|
||||
|
||||
|
||||
def wait(predicate, seconds=60):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if predicate():
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("условие не выполнено за %d с" % seconds)
|
||||
|
||||
|
||||
class DependencyTest(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
compose("start", "postgres", "valkey")
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
|
||||
def test_valkey_down_is_bypassed(self):
|
||||
actor = register("dep")
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
|
||||
compose("stop", "valkey")
|
||||
started = time.time()
|
||||
for _ in range(5):
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
|
||||
self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут")
|
||||
# вход и выход работают без кэша
|
||||
s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
|
||||
self.assertEqual(s[0], 201)
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness
|
||||
|
||||
def test_postgres_down_gives_bounded_503_never_anonymous_access(self):
|
||||
actor = register("dep")
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше
|
||||
compose("stop", "postgres")
|
||||
# Wait past HAProxy's fall=2 health checks: cover the all-unready state,
|
||||
# not just the short interval while the load balancer still routes normally.
|
||||
time.sleep(5)
|
||||
for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]),
|
||||
("GET", "/api/v1/chats", None, actor["token"]),
|
||||
("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]),
|
||||
("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503
|
||||
r = call(method, path, body, token)
|
||||
self.assertEqual(r[0], 503, (path, r))
|
||||
self.assertEqual(r[2]["error"]["code"], "unavailable")
|
||||
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
|
||||
self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"])
|
||||
r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
|
||||
self.assertEqual(r[0], 503)
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 503)
|
||||
self.assertEqual(call("GET", "/health/live")[0], 200)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # без токена — по-прежнему 401
|
||||
compose("start", "postgres")
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2).
|
||||
|
||||
python student-tests/test_expiry.py
|
||||
Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки
|
||||
по умолчанию. Данные и volumes не затрагиваются.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, call, register # noqa: E402
|
||||
|
||||
TTL = 2
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True,
|
||||
capture_output=True, text=True, env=dict(os.environ, **(env or {})))
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if call("GET", "/health/ready")[0] == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("api не стал ready")
|
||||
|
||||
|
||||
class ExpiryTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)})
|
||||
wait_ready()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api")
|
||||
wait_ready()
|
||||
|
||||
def test_session_expires_after_ttl(self):
|
||||
actor = register("exp")
|
||||
session = actor["session"]
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет
|
||||
time.sleep(TTL + 1.5)
|
||||
r = call("GET", "/api/v1/users/me", token=actor["token"])
|
||||
self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)")
|
||||
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
|
||||
self.assertEqual(r[2]["error"]["code"], "unauthorized")
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401)
|
||||
self.assertIn("expires_at", session)
|
||||
|
||||
def test_expiry_is_not_extended_by_use(self):
|
||||
actor = register("exp")
|
||||
for _ in range(3):
|
||||
call("GET", "/api/v1/users/me", token=actor["token"])
|
||||
time.sleep(0.5)
|
||||
time.sleep(1.5)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Идемпотентность отправки сообщений (lab3) и распределение по репликам. Запуск:
|
||||
python student-tests/test_idempotency.py (BASE_URL по умолчанию http://127.0.0.1:8080, через LB)
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import call, psql, register # noqa: E402
|
||||
|
||||
|
||||
def chat_with(owner, *others):
|
||||
ids = [owner["user"]["id"]] + [o["user"]["id"] for o in others]
|
||||
return call("POST", "/api/v1/chats", {"title": "идемпотентность", "member_ids": ids}, owner["token"])[2]
|
||||
|
||||
|
||||
def history(token, chat_id):
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
page = call("GET", "/api/v1/chats/%s/messages?limit=100%s" % (chat_id, "&cursor=" + quote(cursor, safe="") if cursor else ""), token=token)[2]
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
return items
|
||||
|
||||
|
||||
class IdempotencyTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.a, self.b, self.eve = register(), register(), register()
|
||||
self.chat = chat_with(self.a, self.b)
|
||||
self.path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
|
||||
def post(self, actor, body, key, path=None):
|
||||
return call("POST", path or self.path, body, actor["token"], headers={"Idempotency-Key": key})
|
||||
|
||||
def test_key_validation(self):
|
||||
for headers in ({}, {"Idempotency-Key": ""}, {"Idempotency-Key": "a b"},
|
||||
{"Idempotency-Key": "x" * 129}, {"Idempotency-Key": "a/b"}):
|
||||
h = dict(headers)
|
||||
h.setdefault("Content-Type", "application/json")
|
||||
r = call("POST", self.path, raw=b'{"text":"x"}', token=self.a["token"], headers=h)
|
||||
self.assertEqual(r[0], 400, headers)
|
||||
self.assertEqual(r[2]["error"]["code"], "invalid_request")
|
||||
self.assertEqual(self.post(self.a, {"text": "ok"}, "A-z_0.9:-" + "k" * 100)[0], 201)
|
||||
|
||||
def test_replay_returns_same_message_and_creates_no_duplicate(self):
|
||||
key = uuid.uuid4().hex
|
||||
first = self.post(self.a, {"text": "один раз"}, key)
|
||||
second = self.post(self.a, {"text": "один раз"}, key)
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[2], second[2])
|
||||
self.assertEqual(len([m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "один раз"]), 1)
|
||||
|
||||
def test_different_body_same_key_is_409(self):
|
||||
key = uuid.uuid4().hex
|
||||
self.post(self.a, {"text": "первое"}, key)
|
||||
r = self.post(self.a, {"text": "другое"}, key)
|
||||
self.assertEqual(r[0], 409)
|
||||
self.assertEqual(r[2]["error"]["code"], "conflict")
|
||||
|
||||
def test_scope_is_user_and_chat(self):
|
||||
key = uuid.uuid4().hex
|
||||
m1 = self.post(self.a, {"text": "общий ключ"}, key)[2]
|
||||
m2 = self.post(self.b, {"text": "общий ключ"}, key)[2] # другой пользователь
|
||||
other = chat_with(self.a)
|
||||
m3 = self.post(self.a, {"text": "общий ключ"}, key, "/api/v1/chats/%s/messages" % other["id"])[2] # другой чат
|
||||
self.assertEqual(len({m1["id"], m2["id"], m3["id"]}), 3)
|
||||
|
||||
def test_replay_rechecks_session_and_membership(self):
|
||||
key = uuid.uuid4().hex
|
||||
self.post(self.a, {"text": "защищено"}, key)
|
||||
self.assertEqual(self.post(self.eve, {"text": "защищено"}, key)[0], 404) # посторонний
|
||||
r = call("POST", self.path, {"text": "защищено"}, "Z" * 43, headers={"Idempotency-Key": key})
|
||||
self.assertEqual(r[0], 401) # недействительный токен
|
||||
logout = register()
|
||||
chat = chat_with(logout)
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
self.post(logout, {"text": "x"}, key, path)
|
||||
call("DELETE", "/api/v1/auth/sessions/current", token=logout["token"])
|
||||
self.assertEqual(call("POST", path, {"text": "x"}, logout["token"], headers={"Idempotency-Key": key})[0], 401)
|
||||
|
||||
def test_semantically_equal_body_is_replayed(self):
|
||||
key = uuid.uuid4().hex
|
||||
first = self.post(self.a, {"text": "эквивалент"}, key)
|
||||
raw = call("POST", self.path, raw=b'{ "text" : "\u044d\u043a\u0432\u0438\u0432\u0430\u043b\u0435\u043d\u0442" }',
|
||||
token=self.a["token"], headers={"Idempotency-Key": key, "Content-Type": "application/json"})
|
||||
self.assertEqual((raw[0], raw[2]), (201, first[2]))
|
||||
|
||||
def test_concurrent_same_key_creates_exactly_one_message_across_replicas(self):
|
||||
key = uuid.uuid4().hex
|
||||
with ThreadPoolExecutor(24) as pool:
|
||||
results = list(pool.map(lambda _: self.post(self.a, {"text": "гонка ключа"}, key), range(24)))
|
||||
self.assertEqual({r[0] for r in results}, {201})
|
||||
self.assertEqual(len({r[2]["id"] for r in results}), 1)
|
||||
instances = {r[1]["X-Instance-Id"] for r in results}
|
||||
self.assertGreaterEqual(len(instances), 2, "повторы должны обслуживаться разными репликами: %s" % instances)
|
||||
found = [m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "гонка ключа"]
|
||||
self.assertEqual(len(found), 1)
|
||||
|
||||
def test_expired_key_is_reusable_after_ttl(self):
|
||||
key = uuid.uuid4().hex
|
||||
first = self.post(self.a, {"text": "ttl"}, key)[2]
|
||||
psql("UPDATE idempotency_keys SET created_at = now() - interval '25 hours' WHERE idem_key = '%s'" % key)
|
||||
again = self.post(self.a, {"text": "ttl"}, key)
|
||||
self.assertEqual(again[0], 201)
|
||||
self.assertNotEqual(again[2]["id"], first["id"], "после 24 ч ключ считается новым")
|
||||
|
||||
def test_session_is_shared_between_replicas_without_relogin(self):
|
||||
seen = set()
|
||||
for _ in range(30):
|
||||
r = call("GET", "/api/v1/users/me", token=self.a["token"])
|
||||
self.assertEqual(r[0], 200)
|
||||
seen.add(r[1]["X-Instance-Id"])
|
||||
self.assertGreaterEqual(len(seen), 2, seen)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Отказы S3 и согласованность S3 <-> PostgreSQL (уровень 4). Запуск: python student-tests/test_s3_failures.py
|
||||
Тесты останавливают/запускают контейнеры и пересоздают api с внедрением сбоев (volumes сохраняются).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from test_api import ROOT, call, psql, register # noqa: E402
|
||||
from test_attachments import PNG, download, upload # noqa: E402
|
||||
|
||||
|
||||
def compose(*args, env=None, check=True):
|
||||
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, text=True,
|
||||
env=dict(os.environ, **(env or {})), check=check)
|
||||
|
||||
|
||||
def wait(predicate, seconds=90):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if predicate():
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise AssertionError("условие не выполнено за %d с" % seconds)
|
||||
|
||||
|
||||
def recreate_api(env=None):
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api", env=env)
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
out = compose("run", "--rm", "-T", "api", "python", "-m", "app.reconcile", *args).stdout
|
||||
return json.loads(out[out.index("{"):])
|
||||
|
||||
|
||||
def s3_python(code):
|
||||
return compose("exec", "-T", "api", "python", "-c", code).stdout.strip()
|
||||
|
||||
|
||||
class S3FailureTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
compose("start", "seaweedfs", check=False)
|
||||
recreate_api()
|
||||
|
||||
def setUp(self):
|
||||
compose("start", "seaweedfs", check=False)
|
||||
wait(lambda: call("GET", "/health/ready")[0] == 200)
|
||||
self.a = register("s3f")
|
||||
self.chat = call("POST", "/api/v1/chats", {"title": "s3", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
|
||||
|
||||
def test_s3_down_gives_503_text_path_keeps_working(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
count = psql("SELECT count(*) FROM attachments")
|
||||
compose("stop", "seaweedfs")
|
||||
r = upload(self.a["token"], self.chat["id"], PNG)
|
||||
self.assertEqual(r[0], 503, r[2])
|
||||
self.assertEqual(r[2]["error"]["code"], "unavailable")
|
||||
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
|
||||
self.assertEqual(download(self.a["token"], att["id"])[0], 503)
|
||||
self.assertEqual(download(self.a["token"], att["id"], "")[0], 200) # метаданные — из PostgreSQL
|
||||
self.assertEqual(call("POST", "/api/v1/chats/%s/messages" % self.chat["id"], {"text": "текст без S3"}, self.a["token"])[0], 201)
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 200) # readiness не зависит от S3
|
||||
self.assertEqual(psql("SELECT count(*) FROM attachments"), count, "неудачная загрузка не должна оставлять метаданные")
|
||||
compose("start", "seaweedfs")
|
||||
wait(lambda: download(self.a["token"], att["id"])[0] == 200, 90)
|
||||
self.assertEqual(upload(self.a["token"], self.chat["id"], PNG)[0], 201)
|
||||
|
||||
def test_db_failure_after_s3_write_is_compensated(self):
|
||||
recreate_api({"FAULT_INJECTION": "db_after_put"})
|
||||
before = reconcile("--grace", "0")["s3_objects"]
|
||||
r = upload(self.a["token"], self.chat["id"], PNG)
|
||||
self.assertEqual(r[0], 500)
|
||||
self.assertEqual(r[2]["error"]["code"], "internal_error")
|
||||
after = reconcile("--grace", "0")
|
||||
self.assertEqual(after["s3_objects"], before, "объект должен быть удалён компенсацией")
|
||||
self.assertEqual(after["orphan_objects_count"], 0)
|
||||
recreate_api()
|
||||
|
||||
def test_orphan_left_by_failed_compensation_is_found_and_cleaned_safely(self):
|
||||
keeper = upload(self.a["token"], self.chat["id"], PNG)[2] # используемое вложение
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
sent = call("POST", path, {"text": "keeper", "attachment_ids": [keeper["id"]]}, self.a["token"])
|
||||
self.assertEqual(sent[0], 201, sent[2])
|
||||
recreate_api({"FAULT_INJECTION": "db_after_put,s3_delete_fails"})
|
||||
r = upload(self.a["token"], self.chat["id"], PNG)
|
||||
self.assertEqual(r[0], 500)
|
||||
recreate_api()
|
||||
fresh = reconcile("--grace", "900") # grace period: свежий объект не трогаем
|
||||
self.assertGreaterEqual(fresh["orphans_skipped_fresh"], 1)
|
||||
self.assertEqual(fresh["orphan_objects_count"], 0)
|
||||
protected = reconcile("--grace", "900", "--apply")
|
||||
self.assertGreaterEqual(protected["orphans_skipped_fresh"], 1)
|
||||
self.assertEqual(protected["fixed"], {"deleted_orphans": 0, "marked_failed": 0})
|
||||
dry = reconcile("--grace", "0") # dry-run находит, но не удаляет
|
||||
self.assertGreaterEqual(dry["orphan_objects_count"], 1)
|
||||
self.assertEqual(dry["fixed"], {"deleted_orphans": 0, "marked_failed": 0})
|
||||
self.assertEqual(reconcile("--grace", "0")["orphan_objects_count"], dry["orphan_objects_count"]) # dry-run ничего не меняет
|
||||
applied = reconcile("--grace", "0", "--apply")
|
||||
self.assertEqual(applied["fixed"]["deleted_orphans"], dry["orphan_objects_count"])
|
||||
self.assertEqual(reconcile("--grace", "0")["orphan_objects_count"], 0) # повторный запуск идемпотентен
|
||||
self.assertEqual(download(self.a["token"], keeper["id"])[0], 200, "используемое вложение удалять нельзя")
|
||||
|
||||
messages = call("GET", path, token=self.a["token"])[2]["items"]
|
||||
self.assertEqual(messages[0]["id"], sent[2]["id"])
|
||||
self.assertEqual(messages[0]["attachment_ids"], [keeper["id"]])
|
||||
|
||||
def test_missing_object_is_reported_and_attachment_marked_failed(self):
|
||||
att = upload(self.a["token"], self.chat["id"], PNG)[2]
|
||||
key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"])
|
||||
code = ("import asyncio\nfrom app.s3 import S3\nasync def m():\n s=S3(); await s.start(); await s.delete('%s'); await s.stop()\n"
|
||||
"asyncio.run(m())" % key)
|
||||
s3_python(code)
|
||||
dry = reconcile("--grace", "0")
|
||||
self.assertIn(att["id"], dry["missing_objects"])
|
||||
self.assertEqual(psql("SELECT status FROM attachments WHERE id = '%s'" % att["id"]), "ready") # dry-run не меняет
|
||||
reconcile("--grace", "0", "--apply")
|
||||
self.assertEqual(psql("SELECT status || ':' || error_code FROM attachments WHERE id = '%s'" % att["id"]), "failed:object_missing")
|
||||
self.assertEqual(download(self.a["token"], att["id"])[0], 409)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Проверка топологии стенда (уровень 4): число реплик, отсутствие публикации портов API, лимиты ресурсов, healthcheck,
|
||||
отсутствие container_name, единая публичная точка входа. Запуск: python student-tests/test_topology.py"""
|
||||
import json
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
PROJECT = "mtusi-lab4"
|
||||
|
||||
|
||||
def sh(*args):
|
||||
return subprocess.run(args, capture_output=True, text=True, encoding="utf-8").stdout.strip()
|
||||
|
||||
|
||||
def containers(service):
|
||||
ids = sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||||
"--filter", "label=com.docker.compose.service=" + service).split()
|
||||
return [json.loads(sh("docker", "inspect", i))[0] for i in ids]
|
||||
|
||||
|
||||
class TopologyTest(unittest.TestCase):
|
||||
def test_at_least_two_api_replicas_without_published_ports(self):
|
||||
api = containers("api")
|
||||
self.assertGreaterEqual(len(api), 2)
|
||||
for c in api:
|
||||
self.assertFalse([p for p in c["HostConfig"]["PortBindings"] or {}], "реплика API публикует порт в обход LB")
|
||||
self.assertEqual(c["State"]["Health"]["Status"], "healthy")
|
||||
|
||||
def test_only_lb_is_published_and_only_on_localhost(self):
|
||||
published = {}
|
||||
for service in ("api", "postgres", "valkey", "lb", "seaweedfs"):
|
||||
for c in containers(service):
|
||||
for port, binds in (c["HostConfig"]["PortBindings"] or {}).items():
|
||||
published.setdefault(service, []).extend((port, b["HostIp"], b["HostPort"]) for b in binds)
|
||||
self.assertEqual(list(published), ["lb"], published)
|
||||
self.assertEqual(published["lb"], [("8080/tcp", "127.0.0.1", "8080")])
|
||||
|
||||
def test_resource_limits_are_set(self):
|
||||
for service in ("api", "postgres", "valkey", "lb", "seaweedfs"):
|
||||
for c in containers(service):
|
||||
self.assertGreater(c["HostConfig"]["Memory"], 0, service)
|
||||
self.assertGreater(c["HostConfig"]["NanoCpus"], 0, service)
|
||||
|
||||
def test_no_fixed_container_names(self):
|
||||
for c in containers("api"):
|
||||
labels = c["Config"]["Labels"]
|
||||
self.assertEqual(c["Name"].lstrip("/"), "%s-api-%s" % (PROJECT, labels["com.docker.compose.container-number"]))
|
||||
|
||||
def test_api_runs_as_non_root(self):
|
||||
for c in containers("api"):
|
||||
self.assertEqual(c["Config"]["User"], "10001:10001")
|
||||
|
||||
def test_stop_grace_period_covers_drain(self):
|
||||
for c in containers("api"):
|
||||
self.assertGreaterEqual(c["Config"].get("StopTimeout", 10), 15)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user