ЛР 4 — БПИ2302 — Григорьев — на 5 #129
@@ -18,3 +18,15 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
tests
|
||||
student-tests
|
||||
contracts
|
||||
scripts
|
||||
examples
|
||||
*.md
|
||||
Makefile
|
||||
lab.json
|
||||
requirements-dev.txt
|
||||
Dockerfile*
|
||||
.dockerignore
|
||||
.gitignore
|
||||
|
||||
+25
-5
@@ -3,18 +3,38 @@
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
APP_SHUTDOWN_TIMEOUT=5
|
||||
APP_DRAIN_SECONDS=3
|
||||
# Host ports of the load balancer (change to 8090/8405 to run next to another lab stack).
|
||||
LB_PORT=8080
|
||||
STATS_PORT=8404
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
# Credentials are generated locally by: python scripts/init_secrets.py
|
||||
# File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Sessions: PostgreSQL is the source of truth, Valkey is a read-through cache.
|
||||
CACHE_URL=redis://valkey:6379/0
|
||||
SESSION_CACHE_TTL_SECONDS=60
|
||||
REVOCATION_SYNC_SECONDS=1
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
PASSWORD_HASH_CONCURRENCY=4
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_TIMEOUT_SECONDS=2
|
||||
DB_STATEMENT_TIMEOUT_MS=3000
|
||||
DB_REQUEST_TIMEOUT_SECONDS=4
|
||||
# S3 (SeaweedFS). API reads its own limited keys from secret files.
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_TIMEOUT_SECONDS=15
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
UPLOAD_MEMORY_BYTES=1048576
|
||||
UPLOAD_CONCURRENCY=4
|
||||
# Periodic reconciler service: cleans orphans and stale uploads older than the grace period.
|
||||
RECONCILE_INTERVAL_SECONDS=600
|
||||
RECONCILE_GRACE_SECONDS=900
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
*.sh text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
@@ -0,0 +1,31 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
stack:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: cp .env.example .env
|
||||
- run: python scripts/init_secrets.py
|
||||
- run: docker compose config --quiet
|
||||
- run: docker compose up -d --build --wait
|
||||
- run: make test BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: make student-test BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: python student-tests/expiry_check.py --ttl 2
|
||||
- run: python student-tests/failure_check.py
|
||||
- run: python student-tests/consistency_check.py
|
||||
- run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: docker compose start api && sleep 15
|
||||
- run: python student-tests/failover_series.py
|
||||
- run: python student-tests/rolling_update_check.py --no-build
|
||||
- if: failure()
|
||||
run: docker compose ps -a && docker compose logs --no-color
|
||||
- if: always()
|
||||
run: docker compose down -v
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
FROM python:3.12.15-slim-trixie@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_ROOT_USER_ACTION=ignore
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY migrations/ ./migrations/
|
||||
COPY src/ ./src/
|
||||
|
||||
RUN useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=60s --start-interval=1s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"]
|
||||
|
||||
CMD ["python", "src/main.py"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover student-test
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -18,6 +18,9 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
student-test:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -v
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
|
||||
@@ -1,37 +1,196 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Григорьев Павел, БПИ2302
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 5, трек 2 «Согласованность двух хранилищ»
|
||||
- Commit/PR: ветка `solution/bpi2302-grigoriev`; ссылка на PR — после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.3.6 + psycopg-pool 3.3.3, argon2-cffi 25.1.0, redis-py 8.1.0, boto3/botocore 1.44.1, Pillow 13.0.0 (полный список — `requirements.txt`); SeaweedFS 4.48, HAProxy 3.0.30, PostgreSQL 17.11, Valkey 8.1.11; документация AWS S3 (PutObject, ListObjectsV2, SigV4), SeaweedFS (S3 API, identities), PNG specification (chunk CRC, IDAT/zlib, filter types, Adam7), Pillow (draft, DecompressionBomb). При разработке использовался ИИ-ассистент.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требования: Docker Engine 27.x с Compose v2 (проверено: Docker Desktop, Engine 27.2.0, WSL2 на Windows 10); Python 3.9+ для тестов. Порты хоста по умолчанию: `127.0.0.1:8080` (LB) и `127.0.0.1:8404` (статистика HAProxy); меняются переменными `LB_PORT`/`STATS_PORT` в `.env` (на машине автора lab4 запускался на 8090/8405 рядом со стендом lab3). Запуск из копии только Git-файлов на портах по умолчанию проверен двумя холодными стартами с пустыми volumes: `up --wait` — exit 0 за 29 и 40 с, smoke PASS.
|
||||
|
||||
Образы закреплены тегом и digest: `python:3.12.15-slim-trixie@sha256:a6e34c59…` (API), `chrislusf/seaweedfs:4.48@sha256:4e61d15f…`, `haproxy:3.0.30-alpine@sha256:d2b3567d…`, `postgres:17.11-alpine@sha256:266d3c25…`, `valkey/valkey:8.1.11-alpine@sha256:2602ace2…`.
|
||||
|
||||
```sh
|
||||
git clone <URL fork> mtusi2026lab4 && cd mtusi2026lab4
|
||||
git switch solution/bpi2302-grigoriev
|
||||
cp .env.example .env
|
||||
python scripts/init_secrets.py # пароль БД, S3-ключи admin/api/reconciler, secrets/s3_config.json (вне Git)
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --wait # postgres, valkey, s3 → migrate + s3-init (exit 0) → api ×2, reconciler → lb
|
||||
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
|
||||
docker compose start api # вернуть остановленную реплику
|
||||
curl -s -o /dev/null http://127.0.0.1:8080/health/live # один лишний запрос: сдвинуть round robin
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 # теперь — вторая реплика
|
||||
docker compose start api
|
||||
# на Windows — scripts/restart.cmd и scripts/stop-one.cmd
|
||||
python -m unittest discover -s student-tests -v # BASE_URL=... если LB не на 8080
|
||||
python student-tests/consistency_check.py
|
||||
docker compose logs reconciler # периодический сверщик: проход сразу после старта и раз в 10 мин
|
||||
docker compose run --rm reconcile # разовый запуск: dry-run отчёт
|
||||
docker compose run --rm reconcile --apply # разовый запуск: исправить безопасные расхождения
|
||||
python research/track2.py all # трек 2
|
||||
```
|
||||
|
||||
**Bootstrap.** Миграции — одноразовый `migrate` (0001–0005; `0005_attachments` только добавляет таблицы и ослабляет CHECK длины текста — совместима с работающей версией). S3 — одноразовый `s3-init` с admin-ключами: создаёт bucket, только если его нет (иначе `already exists, nothing to do`), делает пробную запись (SeaweedFS выделяет тома при первой записи — без этого первая пользовательская загрузка ждала 11 с), проверяет, что анонимный листинг и GET запрещены. Повтор `docker compose run --rm --no-deps s3-init` безопасен.
|
||||
|
||||
**Healthcheck образа API**: `--interval=10s --start-period=60s --start-interval=1s`. В период старта проверка идёт раз в секунду, поэтому `--wait` не ждёт лишнего; неудачи первые 60 с не делают контейнер unhealthy. Прежний `--start-period=5s` на перегруженной VM (сборка + одновременный старт migrate, s3-init и двух API) однажды привёл к unhealthy через ~35 с и падению `up --wait`.
|
||||
|
||||
**Порядок failover-проверки обеих реплик.** Тест останавливает реплику, обслужившую его первый запрос. Round robin детерминирован, поэтому без вмешательства оба прогона попадают в одну и ту же реплику. Перед вторым прогоном делается один лишний `GET /health/live`: очередь сдвигается, и целью становится вторая реплика (`evidence/failover.txt`).
|
||||
|
||||
Остановка без потери данных: `docker compose stop`. Намеренное удаление учебных данных: `docker compose down -v` (удаляет `pgdata` и `s3data`).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ──► lb (HAProxy) ──► api ×2 ──┬──► postgres : метаданные вложений (+ всё из lab3)
|
||||
├──► s3 (SeaweedFS, приватный bucket) : байты вложений
|
||||
└──► valkey : кэш сессий
|
||||
s3-init (one-shot, admin-ключи) ──► s3 reconciler (раз в 10 мин, ключи сверщика) ──► s3 + postgres
|
||||
```
|
||||
|
||||
| Роль | Service | Версия/digest | Health | Volume | Сеть/порт | Лимиты | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| load_balancer | `lb` | haproxy 3.0.30 | `/lb/health` | — | frontend+backend; `127.0.0.1:${LB_PORT}`, `${STATS_PORT}` | 0.5 CPU, 128M | — |
|
||||
| api ×2 | `api` | сборка из `Dockerfile` | `/health/ready` | — | backend, порт не публикуется | 0.5 CPU, 320M | `database_url`; S3-ключи **api** |
|
||||
| s3 | `s3` | seaweedfs 4.48@sha256:4e61… | `GET :8333/healthz` | `s3data` | backend; наружу из контейнера только S3-порт 8333, мастер/filer/тома привязаны к 127.0.0.1 внутри; ничего не опубликовано на хост | 1 CPU, 512M | `s3_config.json` (identities) |
|
||||
| bootstrap | `s3-init` | образ API | exit 0 | — | backend | 0.5 CPU, 128M | S3-ключи **admin** |
|
||||
| сверщик, периодический | `reconciler` | образ API | файл-пульс обновлён не позже интервал + 2 мин | — | backend | 0.25 CPU, 192M | `database_url`; S3-ключи **reconciler** |
|
||||
| сверщик, разовый запуск | `reconcile` (profile `tools`) | образ API | — | — | backend | 0.5 CPU, 256M | то же |
|
||||
| postgres, valkey, migrate | как в lab3 | | | `pgdata` | backend | | |
|
||||
|
||||
**Почему SeaweedFS.** План был MinIO, но его образы недоступны: на Docker Hub репозиторий `minio/minio` удалён («repository does not exist»), на quay.io — `unauthorized`. Из разрешённых заданием вариантов выбран SeaweedFS (стабильный релиз 4.48 месячной давности). Флаги: `-s3.iam=false` (без встроенного IAM API), `-master.telemetry=false`, бюджеты памяти `-s3.concurrentUploadLimitMB=64`, `-filer.concurrentUploadLimitMB=64`, кэши 16/32 МБ, `GOMEMLIMIT=300MiB` (без них процесс Go держал ~430 МиБ и под нагрузкой упирался в лимит контейнера).
|
||||
|
||||
**Права S3** (файл identities, генерирует `init_secrets.py`):
|
||||
|
||||
| Ключ | Действия | Кому |
|
||||
| --- | --- | --- |
|
||||
| admin | Admin, Read, List, Tagging, Write | только `s3-init` |
|
||||
| api | `Read:messenger-attachments`, `Write:messenger-attachments` | реплики API |
|
||||
| reconciler | `Read`, `List`, `Write` на bucket | сверщик |
|
||||
Ключ API: чтение и запись объектов своего bucket; листинг объектов, создание и удаление bucket, запись в другой bucket — AccessDenied; `ListBuckets` SeaweedFS выполняет, но возвращает пустой список (показываются только bucket'ы с правом List). Анонимной identity нет — анонимные запросы получают 403. Административной консоли наружу нет (UI мастера/filer слушают только localhost внутри контейнера, порты не опубликованы).
|
||||
|
||||
**Схема** (`migrations/0005_attachments.sql`): `attachments(id, chat_id → chats, owner_id → users, filename text 1–255, content_type ∈ {png, jpeg}, size_bytes 1..10485760, sha256 bytea(32), object_key UNIQUE, status ∈ {uploading, ready}, created_at, updated_at)`, индексы `(chat_id)`, `(status, created_at)`; `message_attachments(message_id, attachment_id, position)`, PK `(message_id, attachment_id)`, индекс `(attachment_id)`. Байтов файлов в PostgreSQL нет. Ключ объекта — `attachments/<uuid вложения>`, filename в нём не участвует.
|
||||
|
||||
**Загрузка** (`POST /chats/{id}/attachments?filename=`, сырые байты):
|
||||
1. Сессия (401) → filename 1–255 без `/`, `\`, NUL, управляющих (400) → `Content-Type` только image/png|image/jpeg (415) → членство в чате (404) → `Content-Length` > 10 MiB → **413 до чтения тела**.
|
||||
2. Тело читается потоком с жёстким счётчиком (413 и без Content-Length), SHA-256 на лету, буфер `SpooledTemporaryFile`: до 1 МиБ в RAM, дальше временный файл (удаляется после запроса).
|
||||
3. Не больше 4 одновременных обработок на реплику (`UPLOAD_CONCURRENCY`, ожидание ≤ 30 с, иначе 503).
|
||||
4. Сигнатура содержимого должна совпасть с заявленным типом (иначе **415**); Pillow читает только заголовок — ширина × высота > 40 Мп → **413 без распаковки**; целостность: PNG — собственная потоковая проверка за один проход кусками по 64 КиБ: CRC каждого чанка; IDAT распаковывается `zlib.decompressobj` с `max_length` (вывод не хранится); объём пиксельных данных должен совпасть с вычисленным по IHDR (с учётом Adam7); байт фильтра каждой строки ≤ 4; zlib-поток завершён. Pillow `Image.verify()` не подошёл: читает чанк целиком в память и пиксели не распаковывает. JPEG — декодирование в масштабе 1/8. Битый файл → **415** (в том числе PNG с верными CRC, но мусором или нехваткой данных в IDAT).
|
||||
5. Строка-заготовка `uploading` → S3 PUT → строка `ready` → **201**. S3 недоступен → заготовка удаляется (компенсация), **503**.
|
||||
|
||||
**Согласованность БД и S3.** Порядок «намерение → объект → ready» гарантирует, что у любого объекта, положенного API, строка появилась раньше объекта. Сбой между шагами оставляет заготовку `uploading` (с объектом или без) либо, если потерян ответ PUT после компенсации, объект без строки. Ни то, ни другое не видно пользователям (не ready → 404), и всё это убирает сверщик.
|
||||
|
||||
**Сверщик** `src/reconcile.py` (сервис `reconcile`): листинг объектов по префиксу `attachments/` → снимок таблицы → категории `ok`, `orphan` / `fresh_orphan`, `stale_upload` / `fresh_upload`, `missing`, `size_mismatch`. По умолчанию **dry-run** (JSON-отчёт). `--apply` удаляет только старых сирот (с повторной проверкой, что строки нет) и зависшие заготовки старше **grace period** (по умолчанию 900 с; строка удаляется только если всё ещё `uploading` и не привязана к сообщению, затем её объект). `missing` и `size_mismatch` — только отчёт: восстановить байты неоткуда, а удаление метаданных молча сломало бы сообщения. Повторный запуск идемпотентен. **Автоматический запуск** — постоянный сервис `reconciler`: `reconcile.py --apply --interval 600 --grace-seconds 900` (`RECONCILE_INTERVAL_SECONDS`, `RECONCILE_GRACE_SECONDS`). Первый проход — сразу после старта, дальше раз в 10 мин. Ошибка прохода (S3 или БД недоступны) пишется в лог, следующий проход повторит попытку. По SIGTERM сервис завершается за ~1 с. Healthcheck проверяет файл-пульс, который обновляется после каждого прохода. Так мусор живёт не дольше grace + интервал ≈ 25 мин. Разовые запуски (dry-run по умолчанию) — `docker compose run --rm reconcile`; на время экспериментов трека 2 периодический сервис останавливается, чтобы не вмешаться в состаренные синтетические данные.
|
||||
|
||||
**Скачивание**: проверка членства (404) → поток из S3 кусками по 64 КиБ с `Content-Type` и `Content-Length` оригинала, без redirect; S3 недоступен → 503; метаданные (`GET /attachments/{id}`) отдаются из PostgreSQL и при недоступном S3.
|
||||
|
||||
**Сообщения**: `attachment_ids` 0–10 уникальных UUID (иначе 400); `text=""` допустим при ≥ 1 вложении; все вложения — этого чата и `ready`, иначе **404**, сообщение не создаётся (одна транзакция); любой участник может прикрепить вложение чата; идемпотентный хеш тела включает `attachment_ids`.
|
||||
|
||||
**Readiness** не зависит от S3: при его отказе текстовый чат и метаданные работают, вложения отвечают 503 — исключать реплики из LB из-за отказа файлового хранилища бессмысленно.
|
||||
|
||||
**Перенесено из lab3:** всё (LB, 2 реплики, идемпотентность, drain, отзыв сессий, rolling update, тесты). **Изменено/добавлено:** S3, вложения, сверщик (разовый и периодический); `s3` в restart-хуках; порты LB настраиваются; HAProxy `timeout server 60s` (загрузка 10 МиБ на нагруженном стенде дольше прежних 10 с; у API свои сроки на БД и S3); бюджет ожидания БД перезапускается после чтения тела и ожидания очереди (иначе его съедала передача 10 МиБ).
|
||||
|
||||
Переменные `.env.example` (новые): `LB_PORT`, `STATS_PORT` — хостовые порты; `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`; `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` — ключи API из secrets; `S3_TIMEOUT_SECONDS=15`; `MAX_UPLOAD_BYTES=10485760`; `MAX_IMAGE_PIXELS=40000000`; `UPLOAD_MEMORY_BYTES=1048576` — порог перехода буфера на диск; `UPLOAD_CONCURRENCY=4`; `RECONCILE_INTERVAL_SECONDS=600`, `RECONCILE_GRACE_SECONDS=900` — периодический сверщик. `FAULTS` — только для тестов внедрения сбоев (`after_s3_put`, срабатывает лишь для filename `fault-*`), в рабочей конфигурации пусто.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Файлы — `evidence/`; основной прогон 2026-11-12 со свежих volumes (LB на 8090, стенд lab3 остановлен). После исправлений по итогам проверки 2026-11-13 на том же стенде повторены smoke, student-tests, consistency, persistence, failover, rolling update, upload-memory и track2 scenario; запуск из чистой копии проверен отдельно.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke (upload, message с вложением, download вторым участником, SHA-256, 404 постороннему) | `tests/smoke.py` | PASS | `smoke.txt` |
|
||||
| 3: S3 в стенде, метаданные в PostgreSQL, байты в приватном bucket | `docker compose ps`; `test_storage.py` | s3 healthy, `s3data`; в `attachments` единственный bytea — `sha256`; анонимный GET/листинг напрямую в S3 → 403 | `topology.txt`, `student-tests.txt` |
|
||||
| 3: 201 ready после сохранения, побайтное скачивание, attachment_ids | `test_attachments.py` | PNG и JPEG: 201, ready, sha/size совпадают, скачивание = исходные байты через обе реплики | `student-tests.txt` |
|
||||
| 3: PNG/JPEG ≤ 10 MiB и ≤ 40 Мп, MIME по содержимому, filename, лимиты до чтения/декодирования | `test_attachments.py` | 10 MiB+1 → 413 (по Content-Length и chunked); PNG ~10 MiB → 201; бомба 50 Мп (≈ 50 КБ) → 413; подмена MIME → 415; `/`, `\`, NUL, управляющие, > 255 → 400 | `student-tests.txt` |
|
||||
| 3: upload → остановка обслужившей реплики → download через другую; restart сохраняет | `tests/resilience.py` failover (обе реплики) и persistence (+ s3) | PASS; в логах `Start Seaweed S3 API Server` после рестарта, volumes на месте | `failover.txt`, `persistence.txt` |
|
||||
| 4: большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; без 500 | `test_attachments.py` | 413 / 415 / 415 / 404 / 404; обрезанный PNG, битый CRC, мусор в IDAT и нехватка пикселей при верных CRC, обрезанный JPEG → 415 | `student-tests.txt` |
|
||||
| 4: обрыв загрузки и сбой между S3 и БД не оставляют вечного мусора; механизм с grace period | `consistency_check.py` | обрыв посреди тела → ни строки, ни объекта; S3 down → 503, заготовка удалена компенсацией; сбой после PUT (`FAULTS`) → `uploading` + объект; сверщик: dry-run grace 900 с — не трогает, apply после grace — удаляет оба; сирота: свежая не тронута, старая удалена; повтор — no-op; периодический сервис `reconciler` сам удалил заготовку возрастом 1 ч | `consistency.txt` |
|
||||
| 4: лимиты/потоковая обработка; ≥ 10 одновременных загрузок, пик памяти | `upload_memory.py` | см. «Эксперименты»: 10 и 20 загрузок по 10 MiB — пик памяти процесса реплики ≤ 100 МиБ (не растёт с числом загрузок) | `upload-memory.json` |
|
||||
| 4: bootstrap идемпотентен; отдельные ограниченные S3-ключи; консоль не открыта; приватность вне API | `test_storage.py` | повторный `s3-init` → `already exists`; ключи API: объекты своего bucket — можно; листинг объектов, создание/удаление bucket, запись в чужой — AccessDenied (`ListBuckets` возвращает пустой список); анонимный доступ → 403 | `student-tests.txt` |
|
||||
| недоступность S3 | `consistency_check.py` (2) | upload/download 503 за 0,1 с, метаданные 200, текст 201, ready 200; после старта S3 — 201 | `consistency.txt` |
|
||||
| 5, трек 2 | `research/track2.py` | см. ниже | `track2/` |
|
||||
| сохранено из lab3 | `failover_series.py`, `rolling_update_check.py`, `sigterm_check.py`, `failure_check.py`, `expiry_check.py` | PASS: stop каждой реплики — 0 ошибок, kill — 1–2 × 502 без потерь/дублей; rolling update со сменой образа — 0 ошибок; SIGTERM — 182 запроса после сигнала, все 200 | `failover-series.json`, `rolling-update.json`, `sigterm.txt`, `failure.txt`, `expiry.txt` |
|
||||
|
||||
Всего собственных тестов: 67 (`python -m unittest discover -s student-tests`), все OK.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Пик памяти при одновременных загрузках (уровень 4)
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
`student-tests/upload_memory.py`: PNG ≈ 10 MiB (шум, почти не сжимается) N раз одновременно через LB; перед каждым прогоном реплики пересоздаются; каждые 0,1 с снимаются `memory.current` и `anon`/`file` из `memory.stat` (cgroup v2). `anon` — память процесса, `file` — вытесняемый страничный кэш (в т.ч. временные файлы буфера).
|
||||
|
||||
| Конфигурация | Загрузок OK | Время | Пик `anon` на реплику | Пик `memory.current` | `memory.peak` |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| буфер 1 МиБ, 10 одновременно | 10/10 | 3,0 с | 93–100 МиБ | 151–159 МиБ | 152–160 МиБ |
|
||||
| буфер 1 МиБ, 20 одновременно | 20/20 | 5,3 с | 94–96 МиБ | 189–209 МиБ | 189–211 МиБ |
|
||||
| весь файл в RAM (16 МиБ), 20 одновременно | 20/20 | 5,2 с | 221–250 МиБ | 248–261 МиБ | 249–262 МиБ |
|
||||
|
||||
Замер 2026-11-13, после добавления распаковки IDAT в проверку PNG: память процесса не изменилась (распаковка идёт кусками по 64 КиБ). Первый замер 2026-11-12 (без неё): 93–100 / 94–107 / 232–236 МиБ.
|
||||
|
||||
Память процесса при рабочей настройке **не растёт** с числом одновременных загрузок (очередь из 4 обработок + тела на диске); рост `memory.current` — это вытесняемый кэш временных файлов. При хранении тел в RAM память процесса растёт на ~10 МиБ на загрузку и при лимите 320 МиБ около 30 одновременных загрузок привели бы к OOM. Простой реплики — ~100 МиБ.
|
||||
|
||||
По ходу эксперимента найдено и исправлено: (1) проверка PNG через Pillow `verify()` читала чанк целиком — заменена собственной потоковой проверкой; (2) без ограничения одновременных обработок 20 загрузок упирались в 0,5 CPU реплики и обрывались LB по `timeout server 10s` (504); (3) SeaweedFS без бюджетов памяти упирался в свой лимит и рвал загрузки; (4) бюджет ожидания БД съедала передача тела. Отдельно: когда одновременно работали стенды lab3 и lab4, VM Docker (1,9 ГиБ) ушла в swap (свободно 47 МБ, давление IO 30–40%) — тяжёлые эксперименты выполнены при остановленном lab3.
|
||||
|
||||
### Трек 2. Согласованность PostgreSQL и S3
|
||||
|
||||
**Гипотезы.**
|
||||
- H1 (точность): сверщик находит каждое внедрённое расхождение с правильной категорией и без ложных срабатываний; dry-run ничего не меняет; `--apply` исправляет только безопасные категории; повторный запуск — no-op.
|
||||
- H2 (безопасность): свежие (моложе grace) и используемые в сообщениях вложения не удаляются — в том числе когда сверщик с `--apply` работает во время непрерывных загрузок.
|
||||
- H3 (стоимость): время обхода линейно по числу объектов, память сверщика ограничена.
|
||||
|
||||
**Методика** (`research/track2.py`, `research/s3ops.py`; повторяемо: `python research/track2.py all`). Синтетические строки — в отдельном чате, объекты кладутся/удаляются ключами сверщика из контейнера; grace 30 с.
|
||||
- scenario: 200 ready-вложений (20 привязаны к сообщению) + внедрение: 25 старых сирот; 25 зависших заготовок с объектом и 25 без (возраст 1 ч); 10 пропавших объектов + 5 пропавших у **привязанных** вложений; 10 объектов неверного размера; после ожидания grace — 10 свежих сирот и 10 свежих заготовок. Затем dry-run ×2 → apply → apply → dry-run; плюс настоящее вложение, загруженное через API и привязанное к сообщению.
|
||||
- race: 40 с, 4 потока загрузок через API, сверщик `--apply --grace-seconds 30` в цикле; затем скачивание и сверка каждого подтверждённого вложения.
|
||||
- cost: 3,7 тыс. → 5 тыс. → 10 тыс. → 20 тыс. объектов; dry-run × 3, медиана.
|
||||
|
||||
**Результаты** (`evidence/track2/results.md`, `scenario.json`, `race.json`, `cost.json`):
|
||||
|
||||
| категория | внедрено | найдено dry-run | apply |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| orphan (старше grace) | 25 | 25 | удалено 25 объектов |
|
||||
| fresh_orphan | 10 | 10 | не тронуто |
|
||||
| stale_upload (с объектом / без) | 25 + 25 | 50 | удалено 50 строк и 25 объектов |
|
||||
| fresh_upload | 10 | 10 | не тронуто |
|
||||
| missing (в т.ч. 5 привязанных) | 15 | 15 | только отчёт, строки на месте |
|
||||
| size_mismatch | 10 | 10 | только отчёт |
|
||||
|
||||
Все проверки True: найдено = внедрено; dry-run ничего не меняет (два прогона — одинаковые счётчики и число объектов); apply удалил ровно старых сирот и зависшие заготовки (`skipped: 0`); повторный apply — 0 действий; свежие сироты и заготовки на месте; 20 привязанных строк на месте; настоящее вложение скачивается с исходным SHA-256. **H1 подтверждена.**
|
||||
|
||||
Race: 3 691 загрузка → все 3 691 скачаны без изменений; 5 запусков сверщика с apply ничего не удалили, увидели 5 загрузок «в полёте» и пропустили их как свежие. Первый прогон ошибочно засчитал FAIL: сверщик удалил 20 синтетических «свежих» объектов, оставшихся от scenario и состарившихся за время теста; в исправленной методике состояние перед нагрузкой приводится к чистому. **H2 подтверждена.**
|
||||
|
||||
| объектов | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| 3 719 | 4 | 0,62 | 0,014 | 0,68 | 183 | 90 |
|
||||
| 4 985 | 5 | 0,82 | 0,017 | 0,90 | 180 | 91 |
|
||||
| 9 985 | 10 | 1,54 | 0,072 | 1,71 | 172 | 97 |
|
||||
| 19 985 | 20 | 3,29 | 0,201 | 3,69 | 185 | 111 |
|
||||
|
||||
Время линейно (~180 мкс/объект), ~90 % — листинг S3 (страницы по 1000 ключей); скан БД — доли секунды; память растёт медленно (словари ключей в памяти, ~1 МиБ на 1 000 объектов). **H3 подтверждена.**
|
||||
|
||||
**Ограничения.** Сверщик держит список ключей и строк в памяти (на 20 тыс. — 111 МиБ; для миллионов нужен постраничный обход с сортированным слиянием). Grace защищает только при условии, что загрузка длится меньше grace (у нас ≤ 15 мин против секунд). Источник времени для «свежести» объектов — часы SeaweedFS, для строк — часы PostgreSQL (один хост; в распределённой системе нужен запас на рассинхронизацию). `missing` не исправляется автоматически — это сигнал к восстановлению из бэкапа; загрузка, завершившаяся между листингом S3 и чтением таблицы, может один раз попасть в `missing` (ложное срабатывание только в отчёте, удалений не вызывает; следующий прогон — `ok`). Замеры на одном ноутбуке (Docker Desktop, WSL2); мс — ориентир, категории и счётчики воспроизводятся точно.
|
||||
|
||||
### Отказы и нагрузка
|
||||
|
||||
| Воздействие | Наблюдали |
|
||||
| --- | --- |
|
||||
| S3 остановлен | upload/download 503 за 0,1 с, метаданные 200, текст 201, ready 200; после старта всё работает |
|
||||
| обрыв загрузки клиентом | ни строки, ни объекта |
|
||||
| сбой после S3 PUT до ready | ответ 500 (внедрённый сбой), заготовка + объект → сверщик удаляет после grace |
|
||||
| restart стенда (+ s3) | файл и токен на месте |
|
||||
| остановка реплики, загрузившей файл | файл скачивается через другую |
|
||||
| `docker stop` / `docker kill` каждой реплики под нагрузкой | 0 ошибок / 1–2 × 502, потерь и дублей нет, возврат ~8 с |
|
||||
| rolling update под нагрузкой со сменой образа (`bb7d6d901b97` → `8ed3c1001fea`) | 27,7 с, 0 ошибок, потерь и дублей нет, RPS 207 → 173 → 173 |
|
||||
|
||||
Нагрузка public-скриптом (`tests/load.py --duration 30 --concurrency 4`): 5 158 успешных, 0 ошибок, **171 RPS**, p50 12 / p95 70 / p99 94 мс (стенд lab3 остановлен; при параллельно работающем lab3 в lab3 намерено 65 RPS — VM в swap).
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один узел SeaweedFS без репликации томов, одна БД, один хост — единые точки отказа; бэкап `s3data` не реализован (для PostgreSQL — см. lab2).
|
||||
- Периодический сверщик — один экземпляр; два одновременных прохода безопасны (удаление с повторной проверкой), но запускать несколько экземпляров незачем.
|
||||
- Неиспользуемые ready-вложения (загружены, но не прикреплены) не удаляются — это учтённые данные; политика их хранения не определена контрактом.
|
||||
- Подписанные прямые загрузки (трек 1) не реализованы.
|
||||
- На машине автора VM Docker 1,9 ГиБ: одновременная работа стендов lab3 и lab4 приводит к swap; для экспериментов lab3 останавливался.
|
||||
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
name: grigoriev-lab4
|
||||
|
||||
x-app-env: &app-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
APP_SHUTDOWN_TIMEOUT: ${APP_SHUTDOWN_TIMEOUT:-5}
|
||||
APP_DRAIN_SECONDS: ${APP_DRAIN_SECONDS:-3}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
||||
REVOCATION_SYNC_SECONDS: ${REVOCATION_SYNC_SECONDS:-1}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4}
|
||||
DB_POOL_MIN: ${DB_POOL_MIN:-1}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-2}
|
||||
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-3000}
|
||||
DB_REQUEST_TIMEOUT_SECONDS: ${DB_REQUEST_TIMEOUT_SECONDS:-4}
|
||||
MALLOC_ARENA_MAX: "2"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL: ${CACHE_URL:-redis://valkey:6379/0}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-15}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
UPLOAD_MEMORY_BYTES: ${UPLOAD_MEMORY_BYTES:-1048576}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
|
||||
FAULTS: ${FAULTS:-}
|
||||
|
||||
x-dns-opt: &dns-opt
|
||||
- timeout:1
|
||||
- attempts:1
|
||||
|
||||
services:
|
||||
lb:
|
||||
image: haproxy:3.0.30-alpine@sha256:d2b3567d836f3b7d174edb6d6f611ed5308e7966e53a9cd9a597e0415a298efd
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
- "127.0.0.1:${STATS_PORT:-8404}:8404"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb/health"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 6
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
networks: [frontend, backend]
|
||||
restart: unless-stopped
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
environment:
|
||||
<<: *app-env
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- database_url
|
||||
- {source: s3_api_access_key_id, target: s3_access_key_id}
|
||||
- {source: s3_api_secret_access_key, target: s3_secret_access_key}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_started
|
||||
dns_opt: *dns-opt
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 320M}
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
command: ["python", "src/migrate.py"]
|
||||
environment: *app-env
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
|
||||
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
|
||||
"-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0", "-s3.iam=false",
|
||||
"-s3.config=/run/secrets/s3_config.json",
|
||||
"-s3.concurrentUploadLimitMB=64", "-filer.concurrentUploadLimitMB=64",
|
||||
"-s3.cacheCapacityMB=16", "-s3.readerCacheSizeMB=32",
|
||||
"-master.volumeSizeLimitMB=256", "-volume.max=40", "-master.telemetry=false"]
|
||||
environment:
|
||||
GOMEMLIMIT: 300MiB
|
||||
secrets: [s3_config.json]
|
||||
volumes:
|
||||
- s3data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
|
||||
s3-init:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
command: ["python", "src/s3_bootstrap.py"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
secrets: [database_url, s3_admin_access_key_id, s3_admin_secret_access_key]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
|
||||
reconcile:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
profiles: [tools]
|
||||
entrypoint: ["python", "src/reconcile.py"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
|
||||
S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
|
||||
secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 256M}
|
||||
networks: [backend]
|
||||
|
||||
reconciler:
|
||||
build: .
|
||||
image: mtusi-messenger:lab4
|
||||
command: ["python", "src/reconcile.py", "--apply",
|
||||
"--interval", "${RECONCILE_INTERVAL_SECONDS:-600}", "--grace-seconds", "${RECONCILE_GRACE_SECONDS:-900}"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
|
||||
S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
|
||||
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-600}
|
||||
secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
# жив цикл: файл-пульс обновлялся не позже интервала + 2 мин назад
|
||||
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.path.getmtime('/tmp/reconciler.alive') > float(os.environ['RECONCILE_INTERVAL_SECONDS']) + 120)"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
start_interval: 1s
|
||||
stop_grace_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 192M}
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
|
||||
postgres:
|
||||
image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
command: ["postgres", "-c", "track_io_timing=on"]
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 384M}
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a
|
||||
command: ["valkey-server", "--save", "", "--appendonly", "no",
|
||||
"--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl"]
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 96M}
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
s3data:
|
||||
|
||||
networks:
|
||||
frontend:
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
s3_config.json:
|
||||
file: ./secrets/s3_config.json
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
s3_api_access_key_id:
|
||||
file: ./secrets/s3_api_access_key_id
|
||||
s3_api_secret_access_key:
|
||||
file: ./secrets/s3_api_secret_access_key
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
@@ -3,3 +3,23 @@
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
|
||||
## Файлы этого решения (2026-11-12, свежие volumes, Windows 10 + Docker Desktop 27.2.0, VM 12 CPU / 1,9 GiB, api 2 × 0,5 CPU / 320M, s3 1 CPU / 512M, LB на 127.0.0.1:8090 через `LB_PORT`, стенд lab3 остановлен). После исправлений по итогам проверки 2026-11-13 на том же стенде (не свежие volumes) перегенерированы `topology`, `smoke`, `student-tests`, `consistency`, `persistence`, `failover`, `rolling-update`, `upload-memory`, `track2/scenario.json` и `track2/results.md`
|
||||
|
||||
| Файл | Команда | Что показывает |
|
||||
| --- | --- | --- |
|
||||
| `topology.txt` | `docker compose ps`, `/health/ready` через LB, `docker stats` | две реплики + s3 + периодический `reconciler` + one-shot `migrate`/`s3-init`, порт только у LB, лимиты, лог сверщика |
|
||||
| `smoke.txt` | `python tests/smoke.py` | публичный сценарий lab4: upload, сообщение с вложением, download вторым участником, 404 постороннему |
|
||||
| `persistence.txt` | `tests/resilience.py --mode persistence` + `scripts/restart.cmd` | файл и токен после рестарта postgres/valkey/s3 |
|
||||
| `failover.txt` | `tests/resilience.py --mode failover` + `scripts/stop-one.cmd`, для каждой реплики | файл, загруженный через остановленную реплику, скачивается через другую; перед вторым прогоном один лишний GET, чтобы цель сменилась |
|
||||
| `student-tests.txt` | `python -m unittest discover -s student-tests -v` | 67 тестов: вложения (лимиты, MIME, бомба, повреждения, доступ), приватность bucket и права ключей, всё из lab3 |
|
||||
| `consistency.txt` | `python student-tests/consistency_check.py` | обрыв загрузки, отказ S3 (503 без мусора), сбой после S3 PUT + сверщик с grace, сирота, периодический сервис сам удаляет зависшую заготовку |
|
||||
| `upload-memory.json` | `python student-tests/upload_memory.py` | 10 и 20 одновременных загрузок по 10 MiB: пик памяти реплик (cgroup), буфер на диске vs в RAM |
|
||||
| `failure.txt`, `expiry.txt`, `sigterm.txt` | `failure_check.py`, `expiry_check.py`, `sigterm_check.py` | отказы Valkey/PostgreSQL, истечение сессии, мягкая остановка (из lab3) |
|
||||
| `failover-series.json`, `.txt` | `python student-tests/failover_series.py` | stop и kill каждой реплики под нагрузкой |
|
||||
| `rolling-update.json`, `.txt` | `python student-tests/rolling_update_check.py` | замена реплик под нагрузкой на новый образ, 0 ошибок |
|
||||
| `load.json` | `python tests/load.py --duration 30 --concurrency 4` | базовая нагрузка через LB |
|
||||
| `track2/scenario.json` | `python research/track2.py scenario` | H1: внедрённые расхождения, dry-run/apply/повтор, проверки |
|
||||
| `track2/race.json` | `python research/track2.py race` | H2: сверщик с `--apply` во время непрерывных загрузок |
|
||||
| `track2/cost.json` | `python research/track2.py cost` | H3: время и память сверщика на 3,7–20 тыс. объектов |
|
||||
| `track2/results.md` | `python research/track2.py report` | сводка трека 2 |
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# python student-tests/consistency_check.py
|
||||
# date: 2026-11-13T11:33:26Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090
|
||||
|
||||
1. Обрыв загрузки посреди тела
|
||||
строк не прибавилось (ready/uploading) OK 109/0 -> 109/0
|
||||
2. S3 остановлен: загрузка, скачивание, остальной API
|
||||
upload при остановленном S3 → 503 OK 503 за 0.0 с
|
||||
заготовка удалена компенсацией OK 110/0 -> 110/0
|
||||
download при остановленном S3 → 503 OK 503 за 0.1 с
|
||||
метаданные вложения доступны (они в PostgreSQL) OK
|
||||
текстовое сообщение → 201 OK
|
||||
/health/ready → 200 (S3 не критичен для текстового API) OK
|
||||
после старта S3 upload снова 201 OK
|
||||
3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put)
|
||||
клиент получил ошибку (не 201) OK 500
|
||||
осталась строка uploading и объект в S3 OK 519f7e1c-a7a9-492e-a92a-a1d44f90281e attachments/519f7e1c-a7a9-492e-a92a-a1d44f90281e
|
||||
dry-run, grace 900 с: заготовка свежая, не трогается OK {'ok': 111, 'orphan': 0, 'fresh_orphan': 0, 'stale_upload': 0, 'fresh_upload': 1, 'missing': 0, 'size_mismatch': 0}
|
||||
apply, grace 0: строка и объект удалены OK {'orphan_objects_deleted': 0, 'stale_rows_deleted': 1, 'stale_objects_deleted': 1, 'skipped': 0}
|
||||
объекта больше нет OK
|
||||
4. Объект без строки в БД (orphan)
|
||||
dry-run, grace 900 с: свежая сирота не трогается OK {'ok': 111, 'orphan': 0, 'fresh_orphan': 1, 'stale_upload': 0, 'fresh_upload': 0, 'missing': 0, 'size_mismatch': 0}
|
||||
apply, grace 0: сирота удалена OK {'orphan_objects_deleted': 1, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0}
|
||||
повторный запуск ничего не делает OK {'orphan_objects_deleted': 0, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0}
|
||||
рабочие вложения не пострадали (missing = 0) OK {'ok': 111, 'orphan': 0, 'fresh_orphan': 0, 'stale_upload': 0, 'fresh_upload': 0, 'missing': 0, 'size_mismatch': 0}
|
||||
5. Периодический сервис reconciler убирает зависшую заготовку сам
|
||||
сервис reconciler запущен (docker compose ps) OK
|
||||
заготовка старше grace удалена без ручного запуска OK
|
||||
PASS: сбои загрузки не оставляют неучтённых данных
|
||||
@@ -0,0 +1,10 @@
|
||||
# python student-tests/expiry_check.py --ttl 2
|
||||
# date: 2026-11-12T18:31:48Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090
|
||||
|
||||
пересоздаю api с SESSION_TTL_SECONDS=2
|
||||
сразу после входа: /users/me -> 200
|
||||
TTL ключа этой сессии в Valkey: 1 с (-2 = ключа нет)
|
||||
через 3.5 с: /users/me -> 401, WWW-Authenticate: Bearer realm="messenger"
|
||||
logout истёкшим токеном -> 401
|
||||
PASS: сессия истекла по SESSION_TTL_SECONDS=2, кэш не продлил её
|
||||
возвращаю api с TTL из .env
|
||||
@@ -0,0 +1,162 @@
|
||||
{
|
||||
"replicas": [
|
||||
"f2d964e843f1",
|
||||
"4f6e551ada27"
|
||||
],
|
||||
"warmup_s": 5,
|
||||
"observe_s": 10,
|
||||
"workers": 4,
|
||||
"lb_check": "GET /health/ready every 1s, fall 2, rise 2",
|
||||
"results": [
|
||||
{
|
||||
"replica": "f2d964e843f1",
|
||||
"mode": "stop",
|
||||
"action_seconds": 3.87,
|
||||
"target_served_until_s": 1.6,
|
||||
"first_success_on_other_s": 0.02,
|
||||
"before": {
|
||||
"seconds": 5.0,
|
||||
"ok": 777,
|
||||
"errors": 0,
|
||||
"rps": 155.4,
|
||||
"p50_ms": 14.6,
|
||||
"p95_ms": 79.8,
|
||||
"p99_ms": 104.1,
|
||||
"error_kinds": []
|
||||
},
|
||||
"during_10s": {
|
||||
"seconds": 10.0,
|
||||
"ok": 1090,
|
||||
"errors": 0,
|
||||
"rps": 109.0,
|
||||
"p50_ms": 20.2,
|
||||
"p95_ms": 91.7,
|
||||
"p99_ms": 106.1,
|
||||
"error_kinds": []
|
||||
},
|
||||
"back_in_rotation_s": 7.87,
|
||||
"data": {
|
||||
"confirmed": 2252,
|
||||
"stored": 2252,
|
||||
"lost": 0,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0,
|
||||
"uncertain_at_stop": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"replica": "4f6e551ada27",
|
||||
"mode": "stop",
|
||||
"action_seconds": 4.8,
|
||||
"target_served_until_s": 1.63,
|
||||
"first_success_on_other_s": 0.02,
|
||||
"before": {
|
||||
"seconds": 5.0,
|
||||
"ok": 643,
|
||||
"errors": 0,
|
||||
"rps": 128.6,
|
||||
"p50_ms": 18.0,
|
||||
"p95_ms": 85.4,
|
||||
"p99_ms": 167.4,
|
||||
"error_kinds": []
|
||||
},
|
||||
"during_10s": {
|
||||
"seconds": 10.0,
|
||||
"ok": 499,
|
||||
"errors": 0,
|
||||
"rps": 49.9,
|
||||
"p50_ms": 62.4,
|
||||
"p95_ms": 223.6,
|
||||
"p99_ms": 282.9,
|
||||
"error_kinds": []
|
||||
},
|
||||
"back_in_rotation_s": 8.0,
|
||||
"data": {
|
||||
"confirmed": 1430,
|
||||
"stored": 1430,
|
||||
"lost": 0,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0,
|
||||
"uncertain_at_stop": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"replica": "f2d964e843f1",
|
||||
"mode": "kill",
|
||||
"action_seconds": 0.32,
|
||||
"target_served_until_s": 0.0,
|
||||
"first_success_on_other_s": 0.14,
|
||||
"before": {
|
||||
"seconds": 5.0,
|
||||
"ok": 695,
|
||||
"errors": 3,
|
||||
"rps": 139.0,
|
||||
"p50_ms": 18.6,
|
||||
"p95_ms": 86.7,
|
||||
"p99_ms": 150.6,
|
||||
"error_kinds": [
|
||||
"502"
|
||||
]
|
||||
},
|
||||
"during_10s": {
|
||||
"seconds": 10.0,
|
||||
"ok": 636,
|
||||
"errors": 1,
|
||||
"rps": 63.6,
|
||||
"p50_ms": 31.1,
|
||||
"p95_ms": 104.7,
|
||||
"p99_ms": 1014.3,
|
||||
"error_kinds": [
|
||||
"502"
|
||||
]
|
||||
},
|
||||
"back_in_rotation_s": 8.41,
|
||||
"data": {
|
||||
"confirmed": 1367,
|
||||
"stored": 1367,
|
||||
"lost": 0,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0,
|
||||
"uncertain_at_stop": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"replica": "4f6e551ada27",
|
||||
"mode": "kill",
|
||||
"action_seconds": 0.38,
|
||||
"target_served_until_s": 0.06,
|
||||
"first_success_on_other_s": 0.02,
|
||||
"before": {
|
||||
"seconds": 5.0,
|
||||
"ok": 889,
|
||||
"errors": 0,
|
||||
"rps": 177.8,
|
||||
"p50_ms": 15.4,
|
||||
"p95_ms": 68.3,
|
||||
"p99_ms": 101.3,
|
||||
"error_kinds": []
|
||||
},
|
||||
"during_10s": {
|
||||
"seconds": 10.0,
|
||||
"ok": 745,
|
||||
"errors": 2,
|
||||
"rps": 74.5,
|
||||
"p50_ms": 28.4,
|
||||
"p95_ms": 95.4,
|
||||
"p99_ms": 1010.7,
|
||||
"error_kinds": [
|
||||
"502"
|
||||
]
|
||||
},
|
||||
"back_in_rotation_s": 8.59,
|
||||
"data": {
|
||||
"confirmed": 1675,
|
||||
"stored": 1675,
|
||||
"lost": 0,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0,
|
||||
"uncertain_at_stop": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
f2d964e843f1 stop errors=0 rps 155.4 -> 109.0 target last answer +1.60s restore 7.9s lost=0 dup=0
|
||||
4f6e551ada27 stop errors=0 rps 128.6 -> 49.9 target last answer +1.63s restore 8.0s lost=0 dup=0
|
||||
f2d964e843f1 kill errors=1 rps 139.0 -> 63.6 target last answer +0.00s restore 8.4s lost=0 dup=0
|
||||
4f6e551ada27 kill errors=2 rps 177.8 -> 74.5 target last answer +0.06s restore 8.6s lost=0 dup=0
|
||||
PASS: every replica stopped once per mode; no loss, no duplicates
|
||||
@@ -0,0 +1,20 @@
|
||||
# failover with attachment: tests/resilience.py --mode failover --action-script scripts/stop-one.cmd, for each replica
|
||||
# date: 2026-11-13T11:35:32Z; before run 2 one extra GET /health/live shifts round robin so that the test targets the other replica
|
||||
|
||||
## run 1
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 4.134}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
grigoriev-lab4-api-6 a040ba43e8d1 Up 31 seconds (healthy)
|
||||
grigoriev-lab4-api-5 d42206b660ea Exited (0) 11 seconds ago
|
||||
stopped replica: d42206b660ea
|
||||
restored d42206b660ea
|
||||
|
||||
## run 2
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 4.171}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
grigoriev-lab4-api-6 a040ba43e8d1 Exited (0) 11 seconds ago
|
||||
grigoriev-lab4-api-5 d42206b660ea Up 28 seconds (healthy)
|
||||
stopped replica: a040ba43e8d1
|
||||
restored a040ba43e8d1
|
||||
@@ -0,0 +1,40 @@
|
||||
# python student-tests/failure_check.py
|
||||
# date: 2026-11-12T18:30:53Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090
|
||||
|
||||
1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL
|
||||
первый запрос: обнаружение отказа кэша -> 200 за 0.01 с OK
|
||||
GET /users/me действующим токеном -> 200 за 0.01 с OK
|
||||
POST сообщение -> 201 за 0.01 с OK
|
||||
неизвестный токен -> 401 за 0.01 с OK
|
||||
без Authorization -> 401 за 0.00 с OK
|
||||
/health/ready (Valkey некритичен) -> 200 за 0.01 с OK
|
||||
регистрация и вход без кэша -> 200 за 0.06 с OK
|
||||
logout без кэша -> 204 за 0.01 с OK
|
||||
отозванный токен -> 401 за 0.00 с OK
|
||||
1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша
|
||||
logout при паузе Valkey -> 204 за 0.42 с OK
|
||||
через 6 с после unpause: /users/me -> 401 за 0.03 с OK
|
||||
через 6 с после unpause: сообщение -> 401 за 0.01 с OK
|
||||
1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса
|
||||
logout при паузе Valkey -> 204 за 0.41 с OK
|
||||
после рестарта API и unpause: /users/me -> 401 за 0.02 с OK
|
||||
после рестарта API и unpause: сообщение -> 401 за 0.01 с OK
|
||||
2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит
|
||||
GET история -> 503 за 2.01 с OK
|
||||
POST сообщение -> 503 за 2.00 с OK
|
||||
неизвестный токен (не 200!) -> 503 за 2.01 с OK
|
||||
без Authorization -> 401 за 0.01 с OK
|
||||
вход по Basic верным паролем -> 503 за 2.00 с OK
|
||||
/health/live -> 200 за 0.00 с OK
|
||||
/health/ready -> 503 за 1.00 с OK
|
||||
3. PostgreSQL запущен: готовность через 15.4 с после остановки
|
||||
прежний токен -> 200; история: ['до отказа', 'без кэша']
|
||||
4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают
|
||||
GET история -> 503 за 2.01 с OK
|
||||
POST сообщение -> 503 за 2.00 с OK
|
||||
неизвестный токен (не 200!) -> 503 за 2.01 с OK
|
||||
вход по Basic -> 503 за 2.00 с OK
|
||||
/health/ready -> 503 за 1.00 с OK
|
||||
/health/live -> 200 за 0.00 с OK
|
||||
готовность через 0.2 с после unpause; история: ['до отказа', 'без кэша']
|
||||
PASS: отказы Valkey и PostgreSQL обработаны
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 4,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.114,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 5158,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 171.285,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.787,
|
||||
"p95": 69.894,
|
||||
"p99": 94.394
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# persistence: python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8090
|
||||
# date: 2026-11-13T11:35:32Z; stack after QA fixes (not fresh volumes); host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; api 2 x (0.5 CPU, 320M), s3 1 CPU / 512M
|
||||
|
||||
$ docker compose ps -a (before)
|
||||
NAME STATUS
|
||||
grigoriev-lab4-api-5 Up About a minute (healthy)
|
||||
grigoriev-lab4-api-6 Up About a minute (healthy)
|
||||
grigoriev-lab4-lb-1 Up 3 minutes (healthy)
|
||||
grigoriev-lab4-migrate-1 Exited (0) 3 minutes ago
|
||||
grigoriev-lab4-postgres-1 Up 12 minutes (healthy)
|
||||
grigoriev-lab4-reconciler-1 Up 31 seconds (healthy)
|
||||
grigoriev-lab4-s3-1 Up About a minute (healthy)
|
||||
grigoriev-lab4-s3-init-1 Exited (0) 3 minutes ago
|
||||
grigoriev-lab4-valkey-1 Up 12 minutes (healthy)
|
||||
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
$ docker compose ps -a (after)
|
||||
NAME STATUS
|
||||
grigoriev-lab4-api-5 Up 5 seconds (healthy)
|
||||
grigoriev-lab4-api-6 Up 5 seconds (healthy)
|
||||
grigoriev-lab4-lb-1 Up 4 minutes (healthy)
|
||||
grigoriev-lab4-migrate-1 Exited (0) 7 seconds ago
|
||||
grigoriev-lab4-postgres-1 Up 49 seconds (healthy)
|
||||
grigoriev-lab4-reconciler-1 Up About a minute (healthy)
|
||||
grigoriev-lab4-s3-1 Up 34 seconds (healthy)
|
||||
grigoriev-lab4-s3-init-1 Exited (0) 5 seconds ago
|
||||
grigoriev-lab4-valkey-1 Up 49 seconds (healthy)
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"replicas_before": [
|
||||
"546a297b3e8d",
|
||||
"f00b37e093a8"
|
||||
],
|
||||
"replicas_after": [
|
||||
"b667024b618d",
|
||||
"cfd28971fda4"
|
||||
],
|
||||
"update_seconds": 27.7,
|
||||
"update_exit_code": 0,
|
||||
"before_5s": {
|
||||
"seconds": 5.0,
|
||||
"ok": 1035,
|
||||
"errors": 0,
|
||||
"rps": 207.0,
|
||||
"p50_ms": 10.6,
|
||||
"p95_ms": 60.6,
|
||||
"p99_ms": 68.7,
|
||||
"error_kinds": []
|
||||
},
|
||||
"during_update": {
|
||||
"seconds": 27.71,
|
||||
"ok": 4804,
|
||||
"errors": 0,
|
||||
"rps": 173.4,
|
||||
"p50_ms": 12.6,
|
||||
"p95_ms": 68.0,
|
||||
"p99_ms": 93.0,
|
||||
"error_kinds": []
|
||||
},
|
||||
"after_5s": {
|
||||
"seconds": 5.0,
|
||||
"ok": 864,
|
||||
"errors": 0,
|
||||
"rps": 172.8,
|
||||
"p50_ms": 11.0,
|
||||
"p95_ms": 71.2,
|
||||
"p99_ms": 96.2,
|
||||
"error_kinds": []
|
||||
},
|
||||
"data": {
|
||||
"confirmed": 6708,
|
||||
"stored": 6708,
|
||||
"lost": 0,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0,
|
||||
"uncertain_at_stop": 0
|
||||
},
|
||||
"log": [
|
||||
"old replicas: 546a297b3e8d(image bb7d6d901b97), f00b37e093a8(image bb7d6d901b97)",
|
||||
"migrations applied once",
|
||||
"+ b667024b618d (image 8ed3c1001fea) in rotation after 7.2s",
|
||||
"- 546a297b3e8d stopped and removed after 4.3s",
|
||||
"+ cfd28971fda4 (image 8ed3c1001fea) in rotation after 6.1s",
|
||||
"- f00b37e093a8 stopped and removed after 4.4s",
|
||||
"new replicas: b667024b618d(image 8ed3c1001fea), cfd28971fda4(image 8ed3c1001fea)"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
old replicas: 546a297b3e8d(image bb7d6d901b97), f00b37e093a8(image bb7d6d901b97)
|
||||
migrations applied once
|
||||
+ b667024b618d (image 8ed3c1001fea) in rotation after 7.2s
|
||||
- 546a297b3e8d stopped and removed after 4.3s
|
||||
+ cfd28971fda4 (image 8ed3c1001fea) in rotation after 6.1s
|
||||
- f00b37e093a8 stopped and removed after 4.4s
|
||||
new replicas: b667024b618d(image 8ed3c1001fea), cfd28971fda4(image 8ed3c1001fea)
|
||||
rolling update 27.7s: errors=0, rps 207.0 -> 173.4 -> 172.8, lost=0, duplicates=0, all replaced=True
|
||||
PASS
|
||||
@@ -0,0 +1,16 @@
|
||||
# python student-tests/sigterm_check.py
|
||||
|
||||
сообщение с маркером отправлено: HTTP 201; маркер в логах: False
|
||||
docker stop занял: 3.75 с (лимит 10 с)
|
||||
код выхода контейнера: 0 (0 — сам завершился по SIGTERM; 137 — убит SIGKILL)
|
||||
запросы до SIGTERM: 40, все 200: True
|
||||
запросы после SIGTERM: 182, результаты: ['200']
|
||||
ответов 5xx за всё время: 0
|
||||
|
||||
логи остановки:
|
||||
2026-11-12 18:32:18,839 INFO messenger: signal 15: readiness off, draining for 3.0s
|
||||
2026-11-12 18:32:21,858 INFO uvicorn.error: Shutting down
|
||||
2026-11-12 18:32:21,958 INFO uvicorn.error: Waiting for application shutdown.
|
||||
2026-11-12 18:32:21,958 INFO messenger: application stopping
|
||||
2026-11-12 18:32:21,960 INFO uvicorn.error: Application shutdown complete.
|
||||
2026-11-12 18:32:21,960 INFO uvicorn.error: Finished server process [1]
|
||||
@@ -0,0 +1,4 @@
|
||||
# python tests/smoke.py --base-url http://127.0.0.1:8090
|
||||
# date: 2026-11-13T11:32:48Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090
|
||||
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,146 @@
|
||||
# python -m unittest discover -s student-tests -v
|
||||
# date: 2026-11-13T11:32:48Z; host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; LB 127.0.0.1:8090
|
||||
|
||||
test_duplicate_username (test_api.DuplicateUsernameTest.test_duplicate_username)
|
||||
Повторная регистрация того же username -> 409 conflict ... ok
|
||||
test_empty_text (test_api.EmptyMessageTest.test_empty_text)
|
||||
Пустой текст -> 400 ... ok
|
||||
test_missing_text (test_api.EmptyMessageTest.test_missing_text)
|
||||
Тело без поля text -> 400 ... ok
|
||||
test_text_length_limit (test_api.EmptyMessageTest.test_text_length_limit)
|
||||
4000 символов можно (эмодзи = 1 символ), 4001 — 400 ... ok
|
||||
test_whitespace_text (test_api.EmptyMessageTest.test_whitespace_text)
|
||||
Текст только из пробельных символов -> 400 ... ok
|
||||
test_member_has_access (test_api.ForeignChatTest.test_member_has_access)
|
||||
Участник видит чат (контроль: 404 выше не из-за ошибки в тесте) ... ok
|
||||
test_outsider_cannot_read_chat (test_api.ForeignChatTest.test_outsider_cannot_read_chat)
|
||||
Посторонний читает чат -> 404, как будто чата нет ... ok
|
||||
test_outsider_cannot_write_or_read_history (test_api.ForeignChatTest.test_outsider_cannot_write_or_read_history)
|
||||
Посторонний пишет/читает историю -> 404 ... ok
|
||||
test_outsider_does_not_see_chat_in_list (test_api.ForeignChatTest.test_outsider_does_not_see_chat_in_list)
|
||||
Чужой чат не попадает в список чатов ... ok
|
||||
test_lone_surrogate_is_rejected (test_api.IdentityTest.test_lone_surrogate_is_rejected)
|
||||
Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500 ... ok
|
||||
test_nul_character_is_preserved (test_api.IdentityTest.test_nul_character_is_preserved)
|
||||
U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений ... ok
|
||||
test_sender_cannot_be_forged (test_api.IdentityTest.test_sender_cannot_be_forged)
|
||||
sender_id в теле запрещён: отправитель только из сессии ... ok
|
||||
test_extra_field (test_api.InvalidJsonTest.test_extra_field)
|
||||
Лишнее поле в теле -> 400 ... ok
|
||||
test_json_not_object (test_api.InvalidJsonTest.test_json_not_object)
|
||||
JSON-массив вместо объекта -> 400 ... ok
|
||||
test_malformed_json (test_api.InvalidJsonTest.test_malformed_json)
|
||||
Битый JSON -> 400 invalid_request ... ok
|
||||
test_wrong_type (test_api.InvalidJsonTest.test_wrong_type)
|
||||
Число вместо строки -> 400 (без тихого приведения типов) ... ok
|
||||
test_cursor_from_other_chat (test_api.PaginationTest.test_cursor_from_other_chat)
|
||||
cursor от другого чата -> 400 ... ok
|
||||
test_invalid_cursor_and_limit (test_api.PaginationTest.test_invalid_cursor_and_limit)
|
||||
Мусорный cursor и limit вне 1..100 -> 400 ... ok
|
||||
test_walk_several_pages (test_api.PaginationTest.test_walk_several_pages)
|
||||
7 сообщений по 3: три страницы, без потерь, повторов и перестановок ... ok
|
||||
test_foreign_or_unknown_attachment (test_attachments.MessageAttachmentsTest.test_foreign_or_unknown_attachment)
|
||||
Вложение другого чата или несуществующее → 404, сообщение не создаётся ... ok
|
||||
test_idempotency_includes_attachments (test_attachments.MessageAttachmentsTest.test_idempotency_includes_attachments)
|
||||
Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста ... ok
|
||||
test_invalid_attachment_lists (test_attachments.MessageAttachmentsTest.test_invalid_attachment_lists)
|
||||
Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400 ... ok
|
||||
test_message_with_attachment_only (test_attachments.MessageAttachmentsTest.test_message_with_attachment_only)
|
||||
text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает [] ... ok
|
||||
test_other_member_can_reference_attachment (test_attachments.MessageAttachmentsTest.test_other_member_can_reference_attachment)
|
||||
Боб прикрепляет вложение, загруженное Алисой в этот же чат ... ok
|
||||
test_access_rules (test_attachments.UploadTest.test_access_rules)
|
||||
Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404) ... ok
|
||||
test_corrupted_images (test_attachments.UploadTest.test_corrupted_images)
|
||||
Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415 ... ok
|
||||
test_decompression_bomb (test_attachments.UploadTest.test_decompression_bomb)
|
||||
PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки ... ok
|
||||
test_download_through_every_replica (test_attachments.UploadTest.test_download_through_every_replica)
|
||||
Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей) ... ok
|
||||
test_filename_is_display_only_and_validated (test_attachments.UploadTest.test_filename_is_display_only_and_validated)
|
||||
Unicode-имя сохраняется как есть; нет имени, пустое, /, \, NUL, управляющий символ, >255 → 400 ... ok
|
||||
test_mime_spoofing (test_attachments.UploadTest.test_mime_spoofing)
|
||||
Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415 ... ok
|
||||
test_png_and_jpeg_are_stored_and_downloaded_byte_exact (test_attachments.UploadTest.test_png_and_jpeg_are_stored_and_downloaded_byte_exact)
|
||||
PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type ... ok
|
||||
test_too_large_file (test_attachments.UploadTest.test_too_large_file)
|
||||
10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается ... ok
|
||||
test_wrong_declared_type (test_attachments.UploadTest.test_wrong_declared_type)
|
||||
Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415 ... ok
|
||||
test_authentication_before_validation (test_auth.BearerTest.test_authentication_before_validation)
|
||||
Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400 ... ok
|
||||
test_only_bearer_is_accepted (test_auth.BearerTest.test_only_bearer_is_accepted)
|
||||
Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer ... ok
|
||||
test_scheme_is_case_insensitive (test_auth.BearerTest.test_scheme_is_case_insensitive)
|
||||
Схема аутентификации нечувствительна к регистру (RFC 9110) ... ok
|
||||
test_x_user_id_does_not_override_session (test_auth.BearerTest.test_x_user_id_does_not_override_session)
|
||||
X-User-Id игнорируется: владелец определяется только токеном ... ok
|
||||
test_login_rejects_body (test_auth.LoginTest.test_login_rejects_body)
|
||||
POST /auth/sessions принимает только заголовок, JSON body -> 400 ... ok
|
||||
test_login_returns_opaque_session (test_auth.LoginTest.test_login_returns_opaque_session)
|
||||
Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID ... ok
|
||||
test_malformed_basic (test_auth.LoginTest.test_malformed_basic)
|
||||
Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic ... ok
|
||||
test_password_with_colon_and_unicode (test_auth.LoginTest.test_password_with_colon_and_unicode)
|
||||
Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':' ... ok
|
||||
test_wrong_password_and_unknown_user_look_the_same (test_auth.LoginTest.test_wrong_password_and_unknown_user_look_the_same)
|
||||
Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge ... ok
|
||||
test_logout_revokes_only_current_session (test_auth.LogoutTest.test_logout_revokes_only_current_session)
|
||||
Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает ... ok
|
||||
test_password_is_not_returned (test_auth.RegistrationTest.test_password_is_not_returned)
|
||||
Ответ регистрации и /users/me не содержат password/hash ... ok
|
||||
test_password_length_limits (test_auth.RegistrationTest.test_password_length_limits)
|
||||
Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом ... ok
|
||||
test_password_required_and_not_blank (test_auth.RegistrationTest.test_password_required_and_not_blank)
|
||||
Без password или из одних пробелов -> 400 ... ok
|
||||
test_100_messages_from_10_clients (test_concurrency.ConcurrencyTest.test_100_messages_from_10_clients)
|
||||
100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории ... ok
|
||||
test_parallel_registration_of_same_username (test_concurrency.ConcurrencyTest.test_parallel_registration_of_same_username)
|
||||
20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя ... ok
|
||||
test_chat_creation_is_rolled_back (test_db.DatabaseTest.test_chat_creation_is_rolled_back)
|
||||
Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано ... ok
|
||||
test_history_query_uses_index (test_db.DatabaseTest.test_history_query_uses_index)
|
||||
Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort ... ok
|
||||
test_migrations_are_idempotent (test_db.DatabaseTest.test_migrations_are_idempotent)
|
||||
Повторный запуск миграций ничего не меняет и завершается успешно ... ok
|
||||
test_secrets_are_not_stored_in_plaintext (test_db.DatabaseTest.test_secrets_are_not_stored_in_plaintext)
|
||||
В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен ... ok
|
||||
test_json_formatting_does_not_matter (test_idempotency.IdempotencyTest.test_json_formatting_does_not_matter)
|
||||
Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message ... ok
|
||||
test_key_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_key_scope_is_user_and_chat)
|
||||
Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения ... ok
|
||||
test_missing_or_invalid_key (test_idempotency.IdempotencyTest.test_missing_or_invalid_key)
|
||||
Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400 ... ok
|
||||
test_other_body_with_same_key_conflicts (test_idempotency.IdempotencyTest.test_other_body_with_same_key_conflicts)
|
||||
Тот же ключ с другим телом -> 409, второе сообщение не создаётся ... ok
|
||||
test_parallel_retries_through_replicas_create_one_message (test_idempotency.IdempotencyTest.test_parallel_retries_through_replicas_create_one_message)
|
||||
20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id ... ok
|
||||
test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_access)
|
||||
Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404 ... ok
|
||||
test_replay_returns_same_message (test_idempotency.IdempotencyTest.test_replay_returns_same_message)
|
||||
Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет ... ok
|
||||
test_every_api_response_has_instance_id (test_replicas.ReplicasTest.test_every_api_response_has_instance_id)
|
||||
X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health ... ok
|
||||
test_one_session_is_served_by_several_replicas (test_replicas.ReplicasTest.test_one_session_is_served_by_several_replicas)
|
||||
Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200 ... ok
|
||||
test_write_on_one_replica_is_read_on_another (test_replicas.ReplicasTest.test_write_on_one_replica_is_read_on_another)
|
||||
Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую ... ok
|
||||
test_api_credentials_are_limited (test_storage.StorageTest.test_api_credentials_are_limited)
|
||||
Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied ... ok
|
||||
test_bootstrap_is_idempotent (test_storage.StorageTest.test_bootstrap_is_idempotent)
|
||||
Повторный s3-init: код 0, bucket уже есть, приватность подтверждена ... ok
|
||||
test_bucket_is_private_outside_api (test_storage.StorageTest.test_bucket_is_private_outside_api)
|
||||
Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403 ... ok
|
||||
test_no_original_bytes_in_database_or_api_container (test_storage.StorageTest.test_no_original_bytes_in_database_or_api_container)
|
||||
В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто ... ok
|
||||
test_object_key_is_generated_by_server (test_storage.StorageTest.test_object_key_is_generated_by_server)
|
||||
Ключ объекта = attachments/<uuid вложения>, filename в нём не участвует ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 67 tests in 28.928s
|
||||
|
||||
OK
|
||||
|
||||
отправлено 201: 100, уникальных id: 100, прочитано: 100 за 4 страниц(ы)
|
||||
|
||||
20 повторов: реплики ['6f817671b243', '967308a0d498'], сообщений в истории: 1
|
||||
@@ -0,0 +1,32 @@
|
||||
# topology
|
||||
# date: 2026-11-13T11:47:08Z; stack after QA fixes; LB on 127.0.0.1:8090 (LB_PORT); host: Windows 10, Docker Desktop 27.2.0, VM 12 CPU / 1.9 GiB; api 2 x (0.5 CPU, 320M), s3 1 CPU / 512M, reconciler 0.25 CPU / 192M
|
||||
|
||||
SERVICE NAME STATUS PORTS
|
||||
api grigoriev-lab4-api-5 Up About a minute (healthy)
|
||||
api grigoriev-lab4-api-6 Up About a minute (healthy)
|
||||
lb grigoriev-lab4-lb-1 Up About a minute (healthy) 127.0.0.1:8090->8080/tcp, 127.0.0.1:8405->8404/tcp
|
||||
migrate grigoriev-lab4-migrate-1 Exited (0) About a minute ago
|
||||
postgres grigoriev-lab4-postgres-1 Up 2 minutes (healthy)
|
||||
reconciler grigoriev-lab4-reconciler-1 Up About a minute (healthy)
|
||||
s3 grigoriev-lab4-s3-1 Up 2 minutes (healthy)
|
||||
s3-init grigoriev-lab4-s3-init-1 Exited (0) About a minute ago
|
||||
valkey grigoriev-lab4-valkey-1 Up 2 minutes (healthy)
|
||||
|
||||
$ docker stats --no-stream
|
||||
NAME CPU % MEM USAGE / LIMIT
|
||||
grigoriev-lab4-api-5 0.53% 103MiB / 320MiB
|
||||
grigoriev-lab4-api-6 0.51% 96.47MiB / 320MiB
|
||||
grigoriev-lab4-lb-1 0.17% 17.33MiB / 128MiB
|
||||
grigoriev-lab4-postgres-1 0.07% 32.48MiB / 384MiB
|
||||
grigoriev-lab4-reconciler-1 0.00% 67.75MiB / 192MiB
|
||||
grigoriev-lab4-s3-1 0.33% 161.1MiB / 512MiB
|
||||
grigoriev-lab4-valkey-1 0.68% 13.7MiB / 96MiB
|
||||
|
||||
$ docker compose logs s3-init
|
||||
2026-11-13 11:45:26,442 s3-bootstrap: bucket messenger-attachments already exists, nothing to do
|
||||
2026-11-13 11:45:26,451 s3-bootstrap: warm-up write done in 0.0s (storage volumes allocated before first upload)
|
||||
2026-11-13 11:45:26,469 s3-bootstrap: anonymous list/get denied: bucket messenger-attachments is private
|
||||
|
||||
$ docker compose logs reconciler
|
||||
2026-11-13 11:45:32,048 reconcile: periodic mode: every 600s, apply, grace 900s
|
||||
2026-11-13 11:45:32,366 reconcile: run: objects 165, rows 165, counts {'ok': 165}, actions {'orphan_objects_deleted': 0, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0}, 0.32s
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"repeats": 3,
|
||||
"rows": [
|
||||
{
|
||||
"objects": 3719,
|
||||
"rows": 3734,
|
||||
"list_pages": 4,
|
||||
"list_s3_s": 0.625,
|
||||
"scan_db_s": 0.014,
|
||||
"total_s": 0.679,
|
||||
"max_rss_mib": 89.7,
|
||||
"us_per_object": 182.6
|
||||
},
|
||||
{
|
||||
"objects": 4985,
|
||||
"rows": 5000,
|
||||
"list_pages": 5,
|
||||
"list_s3_s": 0.818,
|
||||
"scan_db_s": 0.017,
|
||||
"total_s": 0.896,
|
||||
"max_rss_mib": 91.4,
|
||||
"us_per_object": 179.7
|
||||
},
|
||||
{
|
||||
"objects": 9985,
|
||||
"rows": 10000,
|
||||
"list_pages": 10,
|
||||
"list_s3_s": 1.539,
|
||||
"scan_db_s": 0.072,
|
||||
"total_s": 1.712,
|
||||
"max_rss_mib": 97.0,
|
||||
"us_per_object": 171.5
|
||||
},
|
||||
{
|
||||
"objects": 19985,
|
||||
"rows": 20000,
|
||||
"list_pages": 20,
|
||||
"list_s3_s": 3.289,
|
||||
"scan_db_s": 0.201,
|
||||
"total_s": 3.692,
|
||||
"max_rss_mib": 111.4,
|
||||
"us_per_object": 184.7
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
{
|
||||
"seconds": 40,
|
||||
"grace_seconds": 30,
|
||||
"preclean_before_load": {
|
||||
"orphan_objects_deleted": 0,
|
||||
"stale_rows_deleted": 0,
|
||||
"stale_objects_deleted": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"reconcile_runs": 5,
|
||||
"uploads_201": 3691,
|
||||
"upload_errors": [],
|
||||
"downloaded_intact": 3691,
|
||||
"deleted_by_reconciler": 0,
|
||||
"fresh_uploads_seen_in_flight": 5,
|
||||
"runs": [
|
||||
{
|
||||
"counts": {
|
||||
"ok": 4101,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 0,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 3,
|
||||
"missing": 248,
|
||||
"size_mismatch": 16276
|
||||
},
|
||||
"actions": {
|
||||
"orphan_objects_deleted": 0,
|
||||
"stale_rows_deleted": 0,
|
||||
"stale_objects_deleted": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"total_s": 5.115
|
||||
},
|
||||
{
|
||||
"counts": {
|
||||
"ok": 4865,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 0,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 1,
|
||||
"missing": 245,
|
||||
"size_mismatch": 16276
|
||||
},
|
||||
"actions": {
|
||||
"orphan_objects_deleted": 0,
|
||||
"stale_rows_deleted": 0,
|
||||
"stale_objects_deleted": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"total_s": 5.269
|
||||
},
|
||||
{
|
||||
"counts": {
|
||||
"ok": 5659,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 0,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 1,
|
||||
"missing": 235,
|
||||
"size_mismatch": 16276
|
||||
},
|
||||
"actions": {
|
||||
"orphan_objects_deleted": 0,
|
||||
"stale_rows_deleted": 0,
|
||||
"stale_objects_deleted": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"total_s": 5.72
|
||||
},
|
||||
{
|
||||
"counts": {
|
||||
"ok": 6359,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 0,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 0,
|
||||
"missing": 188,
|
||||
"size_mismatch": 16276
|
||||
},
|
||||
"actions": {
|
||||
"orphan_objects_deleted": 0,
|
||||
"stale_rows_deleted": 0,
|
||||
"stale_objects_deleted": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"total_s": 5.422
|
||||
},
|
||||
{
|
||||
"counts": {
|
||||
"ok": 7077,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 0,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 0,
|
||||
"missing": 220,
|
||||
"size_mismatch": 16276
|
||||
},
|
||||
"actions": {
|
||||
"orphan_objects_deleted": 0,
|
||||
"stale_rows_deleted": 0,
|
||||
"stale_objects_deleted": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"total_s": 5.409
|
||||
}
|
||||
],
|
||||
"passed": true
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
# Трек 2: результаты
|
||||
|
||||
## H1/H2. Внедрённые сбои (grace 30 с)
|
||||
|
||||
| категория | внедрено | найдено dry-run | действие apply |
|
||||
|---|---|---|---|
|
||||
| orphan | 25 | 25 | удалено объектов: 25 |
|
||||
| fresh_orphan | 10 | 10 | не тронуто |
|
||||
| stale_upload | 50 | 50 | удалено строк: 50, объектов: 25 |
|
||||
| fresh_upload | 10 | 10 | не тронуто |
|
||||
| missing | 15 | 15 | только отчёт |
|
||||
| size_mismatch | 10 | 10 | только отчёт |
|
||||
|
||||
| проверка | результат |
|
||||
|---|---|
|
||||
| detected_equals_injected | True |
|
||||
| dry_run_changes_nothing | True |
|
||||
| apply_deleted_exactly_old_orphans_and_stale | True |
|
||||
| rerun_is_noop | True |
|
||||
| fresh_orphans_kept | True |
|
||||
| fresh_uploads_kept | True |
|
||||
| old_orphans_gone | True |
|
||||
| stale_objects_gone | True |
|
||||
| referenced_rows_kept | True |
|
||||
| missing_still_reported_not_deleted | True |
|
||||
| real_attachment_intact | True |
|
||||
|
||||
## H2. Сверщик под нагрузкой загрузок
|
||||
|
||||
40 с, 4 потока загрузок, apply с grace 30 с в цикле: запусков сверщика 5, загрузок 201 — 3691, скачано без изменений — 3691, удалено сверщиком — 0, «свежих» загрузок в полёте увидено — 5.
|
||||
|
||||
## H3. Стоимость обхода (медиана из 3)
|
||||
|
||||
| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 3719 | 3734 | 4 | 0.62 | 0.014 | 0.68 | 183 | 90 |
|
||||
| 4985 | 5000 | 5 | 0.82 | 0.017 | 0.90 | 180 | 91 |
|
||||
| 9985 | 10000 | 10 | 1.54 | 0.072 | 1.71 | 172 | 97 |
|
||||
| 19985 | 20000 | 20 | 3.29 | 0.201 | 3.69 | 185 | 111 |
|
||||
@@ -0,0 +1,866 @@
|
||||
{
|
||||
"grace_seconds": 30,
|
||||
"injected": {
|
||||
"orphan": 25,
|
||||
"fresh_orphan": 10,
|
||||
"stale_upload": 50,
|
||||
"fresh_upload": 10,
|
||||
"missing": 15,
|
||||
"size_mismatch": 10
|
||||
},
|
||||
"baseline_counts": {
|
||||
"ok": 165,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 0,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 0,
|
||||
"missing": 0,
|
||||
"size_mismatch": 0
|
||||
},
|
||||
"dry_run": {
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 30.0,
|
||||
"objects_listed": 420,
|
||||
"list_pages": 1,
|
||||
"rows_scanned": 425,
|
||||
"counts": {
|
||||
"ok": 340,
|
||||
"orphan": 25,
|
||||
"fresh_orphan": 10,
|
||||
"stale_upload": 50,
|
||||
"fresh_upload": 10,
|
||||
"missing": 15,
|
||||
"size_mismatch": 10
|
||||
},
|
||||
"actions": null,
|
||||
"seconds": {
|
||||
"list_s3": 0.174,
|
||||
"scan_db": 0.006,
|
||||
"total": 0.181
|
||||
},
|
||||
"max_rss_mib": 86.5,
|
||||
"missing": [
|
||||
{
|
||||
"id": "5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"age_s": 67,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"age_s": 67,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"age_s": 67,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"age_s": 67,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"age_s": 67,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
}
|
||||
],
|
||||
"size_mismatch": [
|
||||
{
|
||||
"id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"age_s": 67,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"apply": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 30.0,
|
||||
"objects_listed": 420,
|
||||
"list_pages": 1,
|
||||
"rows_scanned": 425,
|
||||
"counts": {
|
||||
"ok": 340,
|
||||
"orphan": 25,
|
||||
"fresh_orphan": 10,
|
||||
"stale_upload": 50,
|
||||
"fresh_upload": 10,
|
||||
"missing": 15,
|
||||
"size_mismatch": 10
|
||||
},
|
||||
"actions": {
|
||||
"orphan_objects_deleted": 25,
|
||||
"stale_rows_deleted": 50,
|
||||
"stale_objects_deleted": 25,
|
||||
"skipped": 0
|
||||
},
|
||||
"seconds": {
|
||||
"list_s3": 0.136,
|
||||
"scan_db": 0.005,
|
||||
"total": 0.512
|
||||
},
|
||||
"max_rss_mib": 85.5,
|
||||
"missing": [
|
||||
{
|
||||
"id": "5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"age_s": 75,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"age_s": 75,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"age_s": 75,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"age_s": 75,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"age_s": 75,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
}
|
||||
],
|
||||
"size_mismatch": [
|
||||
{
|
||||
"id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"age_s": 75,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"rerun": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 30.0,
|
||||
"objects_listed": 370,
|
||||
"list_pages": 1,
|
||||
"rows_scanned": 375,
|
||||
"counts": {
|
||||
"ok": 340,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 10,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 10,
|
||||
"missing": 15,
|
||||
"size_mismatch": 10
|
||||
},
|
||||
"actions": {
|
||||
"orphan_objects_deleted": 0,
|
||||
"stale_rows_deleted": 0,
|
||||
"stale_objects_deleted": 0,
|
||||
"skipped": 0
|
||||
},
|
||||
"seconds": {
|
||||
"list_s3": 0.167,
|
||||
"scan_db": 0.006,
|
||||
"total": 0.174
|
||||
},
|
||||
"max_rss_mib": 85.7,
|
||||
"missing": [
|
||||
{
|
||||
"id": "5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"age_s": 80,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"age_s": 80,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"age_s": 80,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"age_s": 80,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"age_s": 80,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
}
|
||||
],
|
||||
"size_mismatch": [
|
||||
{
|
||||
"id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"age_s": 80,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"after": {
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 30.0,
|
||||
"objects_listed": 370,
|
||||
"list_pages": 1,
|
||||
"rows_scanned": 375,
|
||||
"counts": {
|
||||
"ok": 340,
|
||||
"orphan": 0,
|
||||
"fresh_orphan": 10,
|
||||
"stale_upload": 0,
|
||||
"fresh_upload": 10,
|
||||
"missing": 15,
|
||||
"size_mismatch": 10
|
||||
},
|
||||
"actions": null,
|
||||
"seconds": {
|
||||
"list_s3": 0.129,
|
||||
"scan_db": 0.005,
|
||||
"total": 0.135
|
||||
},
|
||||
"max_rss_mib": 84.9,
|
||||
"missing": [
|
||||
{
|
||||
"id": "5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"object_key": "attachments/5f1c0d1b-d923-441a-a546-4434f77dde61",
|
||||
"age_s": 84,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"object_key": "attachments/8f3b6fd1-b291-47db-8765-de71de23284e",
|
||||
"age_s": 84,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"object_key": "attachments/b5166ba2-32b2-430d-be01-ae9c0b5d018c",
|
||||
"age_s": 84,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"object_key": "attachments/f4726273-cab8-4ee7-b043-571080b4e5a4",
|
||||
"age_s": 84,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"object_key": "attachments/6f1158a3-ad63-4ac1-8147-df4c27f346cb",
|
||||
"age_s": 84,
|
||||
"referenced": true,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"object_key": "attachments/45a0b2de-7270-4998-a448-578d9b93e75b",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"object_key": "attachments/a0ca6b51-1540-41d9-9bd2-d44191420b6e",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"object_key": "attachments/0923b50f-a2df-46f3-a32a-83b67ac5bc6b",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"object_key": "attachments/e7e0f57a-79a9-41b2-a515-deaddbbdd32a",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"object_key": "attachments/f17c8321-bc41-443a-b77a-e81b2b3aaf57",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"object_key": "attachments/176e6dec-27b7-48cc-925f-26f5439bf2b8",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"object_key": "attachments/c79d7776-1852-4504-ab18-014604a41a36",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"object_key": "attachments/264e2e76-d3b6-479e-8ed1-472467b9eed4",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"object_key": "attachments/fc71784e-0d0a-4283-bb9e-0cce558416e2",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
},
|
||||
{
|
||||
"id": "11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"object_key": "attachments/11ff3a85-8391-45f0-97ef-7644072cc3e8",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": false
|
||||
}
|
||||
],
|
||||
"size_mismatch": [
|
||||
{
|
||||
"id": "2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"object_key": "attachments/2c1a8bd6-839a-41da-9ad1-d81561aef6ab",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"object_key": "attachments/092c24f5-2cc1-4af0-9724-c83e0c4f60b2",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"object_key": "attachments/d3a1b67d-0eed-4658-a358-74cca5ebd991",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"object_key": "attachments/8da2384a-087e-43be-8ab7-ad515655e7a1",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"object_key": "attachments/4a66083e-d775-4085-9870-ffd89cdd89f0",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"object_key": "attachments/1f24119f-719e-4c96-b6c2-a920944c3eec",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"object_key": "attachments/94c09a04-b40a-417b-b953-26a194c2a551",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"object_key": "attachments/187049b9-f44f-43fe-9601-c57c1e93c821",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"object_key": "attachments/0291536d-4e75-4547-8562-7ca0cacb178f",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
},
|
||||
{
|
||||
"id": "c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"object_key": "attachments/c2e14c84-6ab8-4a9f-b26d-301fc3817f31",
|
||||
"age_s": 84,
|
||||
"referenced": false,
|
||||
"object_exists": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"detected": {
|
||||
"orphan": 25,
|
||||
"fresh_orphan": 10,
|
||||
"stale_upload": 50,
|
||||
"fresh_upload": 10,
|
||||
"missing": 15,
|
||||
"size_mismatch": 10
|
||||
},
|
||||
"checks": {
|
||||
"detected_equals_injected": true,
|
||||
"dry_run_changes_nothing": true,
|
||||
"apply_actions": {
|
||||
"orphan_objects_deleted": 25,
|
||||
"stale_rows_deleted": 50,
|
||||
"stale_objects_deleted": 25,
|
||||
"skipped": 0
|
||||
},
|
||||
"apply_deleted_exactly_old_orphans_and_stale": true,
|
||||
"rerun_is_noop": true,
|
||||
"fresh_orphans_kept": true,
|
||||
"fresh_uploads_kept": true,
|
||||
"old_orphans_gone": true,
|
||||
"stale_objects_gone": true,
|
||||
"referenced_rows_kept": true,
|
||||
"missing_still_reported_not_deleted": true,
|
||||
"real_attachment_intact": true
|
||||
},
|
||||
"passed": true
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"runs": [
|
||||
{
|
||||
"label": "spool 1 MiB, 10 parallel",
|
||||
"upload_memory_bytes": 1048576,
|
||||
"parallel": 10,
|
||||
"file_mib": 9.95,
|
||||
"ok": 10,
|
||||
"errors": [],
|
||||
"wall_s": 3.01,
|
||||
"max_request_s": 3.0,
|
||||
"limit_mib": 320,
|
||||
"idle_current_mib": {
|
||||
"32c51c16fad0": 101.2,
|
||||
"b455a3d794e7": 98.2
|
||||
},
|
||||
"peak_since_start_mib": {
|
||||
"32c51c16fad0": 159.8,
|
||||
"b455a3d794e7": 151.8
|
||||
},
|
||||
"sampled_peak_mib": {
|
||||
"32c51c16fad0": {
|
||||
"current": 159.0,
|
||||
"anon": 99.5,
|
||||
"file": 51.7
|
||||
},
|
||||
"b455a3d794e7": {
|
||||
"current": 151.0,
|
||||
"anon": 93.4,
|
||||
"file": 50.0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "spool 1 MiB, 20 parallel",
|
||||
"upload_memory_bytes": 1048576,
|
||||
"parallel": 20,
|
||||
"file_mib": 9.95,
|
||||
"ok": 20,
|
||||
"errors": [],
|
||||
"wall_s": 5.26,
|
||||
"max_request_s": 5.25,
|
||||
"limit_mib": 320,
|
||||
"idle_current_mib": {
|
||||
"a0248524555b": 115.5,
|
||||
"2ce74c7f9f9b": 96.3
|
||||
},
|
||||
"peak_since_start_mib": {
|
||||
"a0248524555b": 211.2,
|
||||
"2ce74c7f9f9b": 189.4
|
||||
},
|
||||
"sampled_peak_mib": {
|
||||
"a0248524555b": {
|
||||
"current": 209.4,
|
||||
"anon": 93.9,
|
||||
"file": 106.7
|
||||
},
|
||||
"2ce74c7f9f9b": {
|
||||
"current": 189.2,
|
||||
"anon": 95.6,
|
||||
"file": 86.3
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"label": "all in RAM, 20 parallel",
|
||||
"upload_memory_bytes": 16777216,
|
||||
"parallel": 20,
|
||||
"file_mib": 9.95,
|
||||
"ok": 20,
|
||||
"errors": [],
|
||||
"wall_s": 5.21,
|
||||
"max_request_s": 5.19,
|
||||
"limit_mib": 320,
|
||||
"idle_current_mib": {
|
||||
"1b0ebeafeae4": 111.6,
|
||||
"5cf20821aa82": 97.8
|
||||
},
|
||||
"peak_since_start_mib": {
|
||||
"1b0ebeafeae4": 248.5,
|
||||
"5cf20821aa82": 261.9
|
||||
},
|
||||
"sampled_peak_mib": {
|
||||
"1b0ebeafeae4": {
|
||||
"current": 247.8,
|
||||
"anon": 221.2,
|
||||
"file": 17.9
|
||||
},
|
||||
"5cf20821aa82": {
|
||||
"current": 261.1,
|
||||
"anon": 250.1,
|
||||
"file": 5.5
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2000
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option http-server-close
|
||||
option forwardfor
|
||||
option redispatch
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 60s
|
||||
timeout check 1s
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 2
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold obsolete 2s
|
||||
hold nx 2s
|
||||
accepted_payload_size 8192
|
||||
|
||||
frontend messenger
|
||||
bind :8080
|
||||
monitor-uri /lb/health
|
||||
use_backend api_live if { nbsrv(api) lt 1 }
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template api 1-4 api:8080
|
||||
|
||||
backend api_live
|
||||
balance roundrobin
|
||||
option httpchk GET /health/live
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
server-template live 1-4 api:8080
|
||||
|
||||
listen stats
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /
|
||||
stats refresh 2s
|
||||
@@ -0,0 +1,46 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_by uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position smallint NOT NULL CHECK (position >= 1),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
ALTER TABLE users
|
||||
DROP CONSTRAINT users_display_name_check,
|
||||
ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'),
|
||||
ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400);
|
||||
|
||||
ALTER TABLE chats
|
||||
DROP CONSTRAINT chats_title_check,
|
||||
ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'),
|
||||
ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400);
|
||||
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_check,
|
||||
ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'),
|
||||
ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000);
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE TABLE session_revocations (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX session_revocations_expires_idx ON session_revocations (expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
scope text NOT NULL,
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash bytea NOT NULL CHECK (octet_length(request_hash) = 32),
|
||||
message_id uuid REFERENCES messages (id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, scope, key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
owner_id uuid NOT NULL REFERENCES users (id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 bytea NOT NULL CHECK (octet_length(sha256) = 32),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
status text NOT NULL CHECK (status IN ('uploading', 'ready')),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
|
||||
CREATE INDEX attachments_status_created_idx ON attachments (status, created_at);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
position smallint NOT NULL CHECK (position >= 1),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_check,
|
||||
ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000);
|
||||
@@ -0,0 +1,27 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
boto3==1.44.1
|
||||
botocore==1.44.1
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
fastapi==0.115.6
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
jmespath==1.1.0
|
||||
pillow==13.0.0
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
pycparser==3.12
|
||||
pydantic==2.10.4
|
||||
pydantic_core==2.27.2
|
||||
python-dateutil==2.9.0.post0
|
||||
redis==8.1.0
|
||||
s3transfer==0.19.3
|
||||
six==1.17.0
|
||||
starlette==0.41.3
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.0
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Операции с S3 для внедрения сбоев в треке 2. Запускается внутри контейнера reconcile (ключи сверщика):
|
||||
python research/s3ops.py put SIZE < keys # создать объекты указанного размера
|
||||
python research/s3ops.py delete < keys # удалить объекты
|
||||
python research/s3ops.py exists < keys # вывести число существующих
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, 'src')
|
||||
|
||||
from botocore.exceptions import ClientError # noqa: E402
|
||||
|
||||
from config import setting, settings # noqa: E402
|
||||
from s3 import make_client # noqa: E402
|
||||
|
||||
client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30)
|
||||
|
||||
|
||||
def exists(key):
|
||||
try:
|
||||
client.head_object(Bucket=settings.s3_bucket, Key=key)
|
||||
return True
|
||||
except ClientError:
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
command = sys.argv[1]
|
||||
keys = [line.strip() for line in sys.stdin if line.strip()]
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
if command == 'put':
|
||||
body = b'x' * int(sys.argv[2])
|
||||
list(pool.map(lambda k: client.put_object(Bucket=settings.s3_bucket, Key=k, Body=body), keys))
|
||||
print(len(keys))
|
||||
elif command == 'delete':
|
||||
list(pool.map(lambda k: client.delete_object(Bucket=settings.s3_bucket, Key=k), keys))
|
||||
print(len(keys))
|
||||
elif command == 'exists':
|
||||
print(sum(pool.map(exists, keys)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Трек 2: согласованность PostgreSQL и S3 — исследование сверщика (src/reconcile.py).
|
||||
|
||||
Стенд запущен (docker compose up -d). BASE_URL — адрес LB (по умолчанию http://127.0.0.1:8080).
|
||||
python research/track2.py scenario # H1/H2: внедрённые сбои → dry-run → apply → повторный apply
|
||||
python research/track2.py race # H2: apply в цикле под непрерывными загрузками
|
||||
python research/track2.py cost # H3: время/память обхода при 1k…20k объектов
|
||||
python research/track2.py cleanup # удалить синтетические данные исследования
|
||||
python research/track2.py report # evidence/track2/results.md
|
||||
python research/track2.py all
|
||||
Синтетические строки живут в отдельном чате пользователя track2_*; cleanup удаляет их и их объекты.
|
||||
На время запуска периодический сервис reconciler останавливается и затем запускается снова.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
|
||||
from helpers import call, compose, new_chat, new_user, sql # noqa: E402
|
||||
from imagegen import SAMPLE_PNG # noqa: E402
|
||||
from test_attachments import download, upload # noqa: E402
|
||||
|
||||
OUT = ROOT / 'evidence' / 'track2'
|
||||
STATE = OUT / 'state.json'
|
||||
GRACE = 30
|
||||
|
||||
|
||||
def save(name, data):
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
(OUT / name).write_text(json.dumps(data, ensure_ascii=False, indent=2, default=str) + '\n', encoding='utf-8')
|
||||
print('saved evidence/track2/' + name)
|
||||
|
||||
|
||||
def load(name):
|
||||
return json.loads((OUT / name).read_text(encoding='utf-8'))
|
||||
|
||||
|
||||
def s3ops(command, keys, *args):
|
||||
result = subprocess.run(['docker', 'compose', 'run', '--rm', '-T', '-v', str(ROOT / 'research') + ':/app/research:ro',
|
||||
'--entrypoint', 'python', 'reconcile', 'research/s3ops.py', command, *map(str, args)],
|
||||
cwd=ROOT, input='\n'.join(keys), capture_output=True, text=True, encoding='utf-8')
|
||||
if result.returncode:
|
||||
raise SystemExit(result.stderr[-2000:])
|
||||
return int(result.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
return json.loads(compose('run', '--rm', 'reconcile', *map(str, args)))
|
||||
|
||||
|
||||
def research_chat():
|
||||
if STATE.exists():
|
||||
return json.loads(STATE.read_text(encoding='utf-8'))
|
||||
user = new_user('track2')
|
||||
chat = new_chat(user)
|
||||
state = {'user_id': user['id'], 'chat_id': chat['id'], 'token': None}
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
STATE.write_text(json.dumps(state), encoding='utf-8')
|
||||
return state
|
||||
|
||||
|
||||
def insert_rows(state, count, status='ready', age_seconds=0, size=1024):
|
||||
"""Синтетические строки attachments в чате исследования; возвращает ключи объектов."""
|
||||
if count <= 0:
|
||||
return []
|
||||
out = sql("WITH new AS (INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"object_key, status, created_at, updated_at) SELECT id, '%s', '%s', 'synthetic.png', 'image/png', %d, "
|
||||
"decode(repeat('00', 32), 'hex'), 'attachments/' || id, '%s', now() - make_interval(secs => %d), now() "
|
||||
"FROM (SELECT gen_random_uuid() AS id FROM generate_series(1, %d)) g RETURNING object_key) "
|
||||
"SELECT string_agg(object_key, ' ') FROM new"
|
||||
% (state['chat_id'], state['user_id'], size, status, age_seconds, count))
|
||||
return out.split()
|
||||
|
||||
|
||||
def reference(state, keys):
|
||||
"""Привязать вложения к сообщению (используемые вложения)."""
|
||||
message = sql("INSERT INTO messages (chat_id, sender_id, text) VALUES ('%s', '%s', convert_to('track2', 'UTF8')) "
|
||||
"RETURNING id" % (state['chat_id'], state['user_id'])).splitlines()[0]
|
||||
sql("INSERT INTO message_attachments (message_id, attachment_id, position) SELECT '%s', a.id, "
|
||||
"row_number() OVER () FROM attachments a WHERE a.object_key IN (%s)"
|
||||
% (message, ','.join("'%s'" % k for k in keys)))
|
||||
|
||||
|
||||
def orphan_keys(count):
|
||||
return ['attachments/%s' % uuid.uuid4() for _ in range(count)]
|
||||
|
||||
|
||||
def scenario():
|
||||
state = research_chat()
|
||||
real = new_user('track2real')
|
||||
real_chat = new_chat(real)
|
||||
real_att = upload(real, real_chat, SAMPLE_PNG)[2]
|
||||
call('POST', '/api/v1/chats/%s/messages' % real_chat['id'], {'text': '', 'attachment_ids': [real_att['id']]},
|
||||
real['token'])
|
||||
baseline = reconcile('--grace-seconds', GRACE)
|
||||
|
||||
ok = insert_rows(state, 200)
|
||||
s3ops('put', ok, 1024)
|
||||
referenced = ok[:20]
|
||||
reference(state, referenced)
|
||||
missing = ok[20:30] + referenced[:5]
|
||||
s3ops('delete', missing)
|
||||
mismatch = ok[40:50]
|
||||
s3ops('put', mismatch, 2048)
|
||||
stale_with_object = insert_rows(state, 25, 'uploading', 3600)
|
||||
s3ops('put', stale_with_object, 1024)
|
||||
stale_without_object = insert_rows(state, 25, 'uploading', 3600)
|
||||
old_orphans = orphan_keys(25)
|
||||
s3ops('put', old_orphans, 1024)
|
||||
print('waiting %ds so that injected orphans become older than grace…' % (GRACE + 3))
|
||||
time.sleep(GRACE + 3)
|
||||
fresh_uploads = insert_rows(state, 10, 'uploading', 0)
|
||||
s3ops('put', fresh_uploads, 1024)
|
||||
fresh_orphans = orphan_keys(10)
|
||||
s3ops('put', fresh_orphans, 1024)
|
||||
|
||||
expected = {'orphan': 25, 'fresh_orphan': 10, 'stale_upload': 50, 'fresh_upload': 10, 'missing': 15,
|
||||
'size_mismatch': 10}
|
||||
base = baseline['counts']
|
||||
dry = reconcile('--grace-seconds', GRACE)
|
||||
dry_again = reconcile('--grace-seconds', GRACE)
|
||||
applied = reconcile('--apply', '--grace-seconds', GRACE)
|
||||
rerun = reconcile('--apply', '--grace-seconds', GRACE)
|
||||
after = reconcile('--grace-seconds', GRACE)
|
||||
detected = {k: dry['counts'][k] - base.get(k, 0) for k in expected}
|
||||
real_download = download(real, real_att['id'])
|
||||
checks = {
|
||||
'detected_equals_injected': detected == expected,
|
||||
'dry_run_changes_nothing': dry_again['counts'] == dry['counts'] and dry_again['objects_listed'] == dry['objects_listed'],
|
||||
'apply_actions': applied['actions'],
|
||||
'apply_deleted_exactly_old_orphans_and_stale': applied['actions'] == {
|
||||
'orphan_objects_deleted': 25, 'stale_rows_deleted': 50, 'stale_objects_deleted': 25, 'skipped': 0},
|
||||
'rerun_is_noop': not any(rerun['actions'][k] for k in ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')),
|
||||
'fresh_orphans_kept': s3ops('exists', fresh_orphans) == 10,
|
||||
'fresh_uploads_kept': s3ops('exists', fresh_uploads) == 10 and after['counts']['fresh_upload'] >= 10,
|
||||
'old_orphans_gone': s3ops('exists', old_orphans) == 0,
|
||||
'stale_objects_gone': s3ops('exists', stale_with_object) == 0,
|
||||
'referenced_rows_kept': int(sql("SELECT count(*) FROM attachments WHERE object_key IN (%s)"
|
||||
% ','.join("'%s'" % k for k in referenced))) == 20,
|
||||
'missing_still_reported_not_deleted': after['counts']['missing'] - base.get('missing', 0) == 15,
|
||||
'real_attachment_intact': real_download[0] == 200 and hashlib.sha256(real_download[2]).hexdigest() == real_att['sha256'],
|
||||
}
|
||||
passed = all(v for k, v in checks.items() if k != 'apply_actions')
|
||||
save('scenario.json', {'grace_seconds': GRACE, 'injected': expected, 'baseline_counts': base,
|
||||
'dry_run': dry, 'apply': applied, 'rerun': rerun, 'after': after,
|
||||
'detected': detected, 'checks': checks, 'passed': passed})
|
||||
for name, value in checks.items():
|
||||
print(' %-48s %s' % (name, value))
|
||||
print('PASS' if passed else 'FAIL')
|
||||
|
||||
|
||||
def race(seconds=40):
|
||||
user = new_user('track2race')
|
||||
chat = new_chat(user)
|
||||
uploaded, errors, stop = [], [], threading.Event()
|
||||
|
||||
def uploader():
|
||||
while not stop.is_set():
|
||||
status, _, body = upload(user, chat, SAMPLE_PNG)
|
||||
(uploaded.append(body['id']) if status == 201 else errors.append(status))
|
||||
|
||||
preclean = reconcile('--apply', '--grace-seconds', 0)['actions']
|
||||
threads = [threading.Thread(target=uploader) for _ in range(4)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
runs, started = [], time.monotonic()
|
||||
while time.monotonic() - started < seconds:
|
||||
report = reconcile('--apply', '--grace-seconds', GRACE)
|
||||
runs.append({'counts': report['counts'], 'actions': report['actions'], 'total_s': report['seconds']['total']})
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
intact = 0
|
||||
for attachment_id in uploaded:
|
||||
status, _, body = download(user, attachment_id)
|
||||
intact += status == 200 and body == SAMPLE_PNG
|
||||
result = {'seconds': seconds, 'grace_seconds': GRACE, 'preclean_before_load': preclean,
|
||||
'reconcile_runs': len(runs), 'uploads_201': len(uploaded),
|
||||
'upload_errors': errors, 'downloaded_intact': intact,
|
||||
'deleted_by_reconciler': sum(r['actions']['orphan_objects_deleted'] + r['actions']['stale_rows_deleted']
|
||||
for r in runs),
|
||||
'fresh_uploads_seen_in_flight': sum(r['counts']['fresh_upload'] for r in runs), 'runs': runs}
|
||||
result['passed'] = intact == len(uploaded) and result['deleted_by_reconciler'] == 0
|
||||
save('race.json', result)
|
||||
print('race: %d uploads, %d intact, reconciler runs %d, deleted %d, in-flight uploads seen %d → %s'
|
||||
% (len(uploaded), intact, len(runs), result['deleted_by_reconciler'], result['fresh_uploads_seen_in_flight'],
|
||||
'PASS' if result['passed'] else 'FAIL'))
|
||||
|
||||
|
||||
def cost(sizes=(1000, 5000, 10000, 20000), repeats=3):
|
||||
state = research_chat()
|
||||
rows = []
|
||||
for target in sizes:
|
||||
current = int(sql("SELECT count(*) FROM attachments"))
|
||||
if target > current:
|
||||
keys = insert_rows(state, target - current)
|
||||
started = time.monotonic()
|
||||
for start in range(0, len(keys), 2000):
|
||||
s3ops('put', keys[start:start + 2000], 256)
|
||||
print(' +%d objects in %.1fs' % (len(keys), time.monotonic() - started))
|
||||
reports = [reconcile('--grace-seconds', GRACE) for _ in range(repeats)]
|
||||
row = {'objects': reports[-1]['objects_listed'], 'rows': reports[-1]['rows_scanned'],
|
||||
'list_pages': reports[-1]['list_pages'],
|
||||
'list_s3_s': statistics.median(r['seconds']['list_s3'] for r in reports),
|
||||
'scan_db_s': statistics.median(r['seconds']['scan_db'] for r in reports),
|
||||
'total_s': statistics.median(r['seconds']['total'] for r in reports),
|
||||
'max_rss_mib': max(r['max_rss_mib'] for r in reports)}
|
||||
row['us_per_object'] = round(row['total_s'] / row['objects'] * 1e6, 1)
|
||||
rows.append(row)
|
||||
print(' %(objects)6d objects: list %(list_s3_s).2fs, db %(scan_db_s).3fs, total %(total_s).2fs, '
|
||||
'rss %(max_rss_mib).0f MiB, %(us_per_object).0f µs/object' % row)
|
||||
save('cost.json', {'repeats': repeats, 'rows': rows})
|
||||
|
||||
|
||||
def cleanup():
|
||||
if not STATE.exists():
|
||||
print('nothing to clean')
|
||||
return
|
||||
state = json.loads(STATE.read_text(encoding='utf-8'))
|
||||
keys = sql("SELECT coalesce(string_agg(object_key, ' '), '') FROM attachments WHERE chat_id = '%s'" % state['chat_id']).split()
|
||||
for start in range(0, len(keys), 2000):
|
||||
s3ops('delete', keys[start:start + 2000])
|
||||
sql("DELETE FROM message_attachments WHERE message_id IN (SELECT id FROM messages WHERE chat_id = '%s')" % state['chat_id'])
|
||||
sql("DELETE FROM messages WHERE chat_id = '%s'" % state['chat_id'])
|
||||
sql("DELETE FROM attachments WHERE chat_id = '%s'" % state['chat_id'])
|
||||
leftovers = reconcile('--apply', '--grace-seconds', 0)
|
||||
STATE.unlink()
|
||||
print('cleanup: %d research objects deleted; reconciler afterwards: %s' % (len(keys), leftovers['counts']))
|
||||
|
||||
|
||||
def report():
|
||||
s, r, c = load('scenario.json'), load('race.json'), load('cost.json')
|
||||
lines = ['# Трек 2: результаты', '', '## H1/H2. Внедрённые сбои (grace %d с)' % s['grace_seconds'], '',
|
||||
'| категория | внедрено | найдено dry-run | действие apply |', '|---|---|---|---|']
|
||||
actions = {'orphan': 'удалено объектов: %d' % s['apply']['actions']['orphan_objects_deleted'],
|
||||
'stale_upload': 'удалено строк: %d, объектов: %d' % (s['apply']['actions']['stale_rows_deleted'],
|
||||
s['apply']['actions']['stale_objects_deleted']),
|
||||
'fresh_orphan': 'не тронуто', 'fresh_upload': 'не тронуто', 'missing': 'только отчёт',
|
||||
'size_mismatch': 'только отчёт'}
|
||||
for name, count in s['injected'].items():
|
||||
lines.append('| %s | %d | %d | %s |' % (name, count, s['detected'][name], actions[name]))
|
||||
lines += ['', '| проверка | результат |', '|---|---|']
|
||||
lines += ['| %s | %s |' % (k, v) for k, v in s['checks'].items() if k != 'apply_actions']
|
||||
lines += ['', '## H2. Сверщик под нагрузкой загрузок', '',
|
||||
'%d с, 4 потока загрузок, apply с grace %d с в цикле: запусков сверщика %d, загрузок 201 — %d, '
|
||||
'скачано без изменений — %d, удалено сверщиком — %d, «свежих» загрузок в полёте увидено — %d.'
|
||||
% (r['seconds'], r['grace_seconds'], r['reconcile_runs'], r['uploads_201'], r['downloaded_intact'],
|
||||
r['deleted_by_reconciler'], r['fresh_uploads_seen_in_flight']),
|
||||
'', '## H3. Стоимость обхода (медиана из %d)' % c['repeats'], '',
|
||||
'| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |',
|
||||
'|---|---|---|---|---|---|---|---|']
|
||||
for row in c['rows']:
|
||||
lines.append('| %(objects)d | %(rows)d | %(list_pages)d | %(list_s3_s).2f | %(scan_db_s).3f | %(total_s).2f | '
|
||||
'%(us_per_object).0f | %(max_rss_mib).0f |' % row)
|
||||
(OUT / 'results.md').write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
print('saved evidence/track2/results.md')
|
||||
|
||||
|
||||
COMMANDS = {'scenario': scenario, 'race': race, 'cost': cost, 'cleanup': cleanup, 'report': report}
|
||||
|
||||
if __name__ == '__main__':
|
||||
command = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||
# Периодический сервис reconciler (--apply, grace 900 с) удалил бы состаренные синтетические заготовки
|
||||
# посреди эксперимента; на время исследования он остановлен, сверщик запускается явно.
|
||||
paused = 'reconciler' in compose('ps', '--status', 'running', '--services')
|
||||
if paused:
|
||||
compose('stop', 'reconciler')
|
||||
try:
|
||||
for step in (['scenario', 'race', 'cost', 'report', 'cleanup'] if command == 'all' else [command]):
|
||||
print('==', step)
|
||||
COMMANDS[step]()
|
||||
finally:
|
||||
if paused:
|
||||
compose('start', 'reconciler')
|
||||
@@ -0,0 +1,38 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / 'secrets'
|
||||
|
||||
|
||||
def write(name, value):
|
||||
path = SECRETS / name
|
||||
if path.exists():
|
||||
print('keep existing secrets/%s' % name)
|
||||
return path.read_text(encoding='utf-8').strip()
|
||||
path.write_text(value + '\n', encoding='utf-8')
|
||||
os.chmod(path, 0o644)
|
||||
print('created secrets/%s' % name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
password = write('postgres_password', secrets.token_hex(24))
|
||||
write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password)
|
||||
bucket = 'messenger-attachments'
|
||||
rights = {'admin': ['Admin', 'Read', 'List', 'Tagging', 'Write'],
|
||||
'api': ['Read:' + bucket, 'Write:' + bucket],
|
||||
'reconciler': ['Read:' + bucket, 'List:' + bucket, 'Write:' + bucket]}
|
||||
identities = []
|
||||
for role, actions in rights.items():
|
||||
key = write('s3_%s_access_key_id' % role, role + '-' + secrets.token_hex(8))
|
||||
secret = write('s3_%s_secret_access_key' % role, secrets.token_hex(24))
|
||||
identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions})
|
||||
write('s3_config.json', json.dumps({'identities': identities}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop api || exit /b 1
|
||||
docker compose restart postgres valkey s3 || exit /b 1
|
||||
docker compose up -d --no-build --no-recreate --wait postgres valkey s3 || exit /b 1
|
||||
docker compose start api || exit /b 1
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop api
|
||||
docker compose restart postgres valkey s3
|
||||
docker compose up -d --no-build --no-recreate --wait postgres valkey s3
|
||||
docker compose start api
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Поочерёдная замена API-реплик без остановки сервиса.
|
||||
|
||||
python scripts/rolling_update.py # пересобрать образ и заменить реплики по одной
|
||||
python scripts/rolling_update.py --no-build # заменить на текущий образ
|
||||
|
||||
1. Один раз применяет миграции (отдельный контейнер migrate; совместимые, только добавляющие изменения).
|
||||
2. Для каждой старой реплики: поднимает одну новую (--scale +1, --no-recreate), ждёт, пока LB начнёт
|
||||
отправлять на неё трафик, затем штатно останавливает старую (SIGTERM → вывод из ротации → завершение)
|
||||
и удаляет её контейнер. Число реплик в ротации не опускается ниже исходного.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def running_api():
|
||||
return compose('ps', '-q', 'api').split()
|
||||
|
||||
|
||||
def instance_of_next_response():
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + '/health/ready', timeout=2) as response:
|
||||
return response.headers.get('X-Instance-Id')
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def wait_in_rotation(container, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
if instance_of_next_response() == container[:12]:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise SystemExit('new replica %s did not enter LB rotation' % container[:12])
|
||||
|
||||
|
||||
def image_of(container):
|
||||
return subprocess.run(['docker', 'inspect', '-f', '{{.Image}}', container], capture_output=True, text=True).stdout.strip()[7:19]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--no-build', action='store_true')
|
||||
args = parser.parse_args()
|
||||
old = running_api()
|
||||
print('old replicas:', ', '.join('%s(image %s)' % (c[:12], image_of(c)) for c in old))
|
||||
if not args.no_build:
|
||||
compose('build', 'api')
|
||||
compose('run', '--rm', '--no-deps', 'migrate')
|
||||
print('migrations applied once')
|
||||
for container in old:
|
||||
before = set(running_api())
|
||||
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (len(before) + 1), 'api')
|
||||
new = (set(running_api()) - before).pop()
|
||||
started = time.monotonic()
|
||||
wait_in_rotation(new)
|
||||
print('+ %s (image %s) in rotation after %.1fs' % (new[:12], image_of(new), time.monotonic() - started))
|
||||
started = time.monotonic()
|
||||
subprocess.run(['docker', 'stop', '-t', '15', container], check=True, capture_output=True)
|
||||
subprocess.run(['docker', 'rm', container], check=True, capture_output=True)
|
||||
print('- %s stopped and removed after %.1fs' % (container[:12], time.monotonic() - started))
|
||||
print('new replicas:', ', '.join('%s(image %s)' % (c[:12], image_of(c)) for c in running_api()))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,11 @@
|
||||
@echo off
|
||||
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
|
||||
cd /d "%~dp0.."
|
||||
for /f %%i in ('docker compose ps -q api') do (
|
||||
echo %%i| findstr /b /i "%TARGET_INSTANCE_ID%" >nul && (
|
||||
docker stop -t 10 %%i >nul
|
||||
exit /b 0
|
||||
)
|
||||
)
|
||||
echo no running api replica matches TARGET_INSTANCE_ID 1>&2
|
||||
exit /b 1
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
for id in $(docker compose ps -q api); do
|
||||
case "$id" in
|
||||
"$TARGET_INSTANCE_ID"*)
|
||||
docker stop -t 10 "$id" >/dev/null
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
echo "no running api replica matches TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
+252
@@ -0,0 +1,252 @@
|
||||
from contextlib import asynccontextmanager
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
import tempfile
|
||||
import threading
|
||||
from typing import Annotated, Literal, Optional
|
||||
import uuid
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, Query, Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
import psycopg
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
import auth
|
||||
from cache import cache
|
||||
from config import settings
|
||||
from db import connection, pool, request_deadline, restart_budget
|
||||
from errors import ApiError, error_response
|
||||
import images
|
||||
from models import CreateChat, CreateMessage, CreateUser
|
||||
import s3
|
||||
import store
|
||||
|
||||
log = logging.getLogger('messenger')
|
||||
|
||||
Limit = Annotated[int, Query(ge=1, le=100)]
|
||||
Cursor = Annotated[Optional[str], Query(min_length=1)]
|
||||
|
||||
INSTANCE_ID = socket.gethostname()
|
||||
IDEMPOTENCY_KEY = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
|
||||
|
||||
PUBLIC = {('POST', '/api/v1/users'), ('POST', '/api/v1/auth/sessions')}
|
||||
|
||||
state = {'ready': False}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
pool.open(wait=False)
|
||||
log.info('instance %s, session cache: %s', INSTANCE_ID, 'valkey' if cache.client else 'disabled')
|
||||
state['ready'] = True
|
||||
log.info('application started')
|
||||
yield
|
||||
state['ready'] = False
|
||||
log.info('application stopping')
|
||||
pool.close(timeout=2)
|
||||
|
||||
|
||||
app = FastAPI(title='MTUSI Messenger lab4', lifespan=lifespan)
|
||||
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_context(request: Request, call_next):
|
||||
request.state.request_id = uuid.uuid4().hex
|
||||
request_deadline.set({'deadline': None})
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
log.exception('unhandled error request_id=%s', request.state.request_id)
|
||||
response = error_response(request, 500, 'Внутренняя ошибка сервера')
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = INSTANCE_ID
|
||||
log.info('%s %s -> %d %.1fms request_id=%s', request.method, request.url.path,
|
||||
response.status_code, (time.perf_counter() - started) * 1000, request.state.request_id)
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(request: Request, exc: ApiError):
|
||||
return error_response(request, exc.status, exc.message, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def handle_http_error(request: Request, exc: StarletteHTTPException):
|
||||
messages = {404: 'Ресурс не найден', 405: 'Метод не поддерживается'}
|
||||
return error_response(request, exc.status_code, messages.get(exc.status_code, 'Ошибка запроса'))
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def handle_validation_error(request: Request, exc: RequestValidationError):
|
||||
if request.url.path.startswith('/api/v1/') and (request.method, request.url.path) not in PUBLIC:
|
||||
try:
|
||||
await run_in_threadpool(auth.authenticate, request.headers.get('authorization'))
|
||||
except ApiError as error:
|
||||
return error_response(request, error.status, error.message, error.headers)
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError):
|
||||
return error_response(request, 503, 'Хранилище временно недоступно')
|
||||
errors = exc.errors()
|
||||
where = '.'.join(str(part) for part in errors[0]['loc']) if errors else 'request'
|
||||
return error_response(request, 400, 'Некорректный запрос: ' + where)
|
||||
|
||||
|
||||
@app.exception_handler(psycopg.OperationalError)
|
||||
@app.exception_handler(psycopg.InterfaceError)
|
||||
async def handle_database_unavailable(request: Request, exc: psycopg.Error):
|
||||
log.warning('database unavailable: %s request_id=%s', type(exc).__name__, request.state.request_id)
|
||||
return error_response(request, 503, 'Хранилище временно недоступно')
|
||||
|
||||
|
||||
@app.get('/health/live')
|
||||
async def liveness():
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
@app.get('/health/ready')
|
||||
def readiness():
|
||||
if not state['ready']:
|
||||
raise ApiError(503, 'Сервис не готов')
|
||||
with connection(limit=1) as conn:
|
||||
conn.execute('SELECT 1')
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
def current_session(authorization: Annotated[Optional[str], Header()] = None):
|
||||
return auth.authenticate(authorization)
|
||||
|
||||
|
||||
def current_user(session: Annotated[dict, Depends(current_session)]):
|
||||
return session['user']
|
||||
|
||||
|
||||
@app.post('/api/v1/users', status_code=201)
|
||||
def register_user(body: CreateUser):
|
||||
return store.create_user(body.username, body.display_name, auth.hash_password(body.password))
|
||||
|
||||
|
||||
@app.post('/api/v1/auth/sessions', status_code=201)
|
||||
def create_session(request: Request, authorization: Annotated[Optional[str], Header()] = None):
|
||||
if request.headers.get('content-length', '0') != '0' or 'transfer-encoding' in request.headers:
|
||||
raise ApiError(400, 'Вход принимает только заголовок Authorization: Basic, без тела')
|
||||
return auth.login(authorization)
|
||||
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
def delete_session(session: Annotated[dict, Depends(current_session)]):
|
||||
auth.logout(session)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get('/api/v1/users/me')
|
||||
def get_me(user: Annotated[dict, Depends(current_user)]):
|
||||
return user
|
||||
|
||||
|
||||
@app.post('/api/v1/chats', status_code=201)
|
||||
def create_chat(body: CreateChat, user: Annotated[dict, Depends(current_user)]):
|
||||
return store.create_chat(user['id'], body.title, body.member_ids)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats')
|
||||
def list_chats(user: Annotated[dict, Depends(current_user)], limit: Limit = 50, cursor: Cursor = None):
|
||||
return store.list_chats(user['id'], cursor, limit)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}')
|
||||
def get_chat(chat_id: uuid.UUID, user: Annotated[dict, Depends(current_user)]):
|
||||
return store.get_chat(str(chat_id), user['id'])
|
||||
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
|
||||
def create_message(chat_id: uuid.UUID, body: CreateMessage, user: Annotated[dict, Depends(current_user)],
|
||||
idempotency_key: Annotated[Optional[str], Header()] = None):
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY.match(idempotency_key):
|
||||
raise ApiError(400, 'Нужен заголовок Idempotency-Key: [A-Za-z0-9._:-]{1,128}')
|
||||
canonical = json.dumps({'text': body.text, 'attachment_ids': body.attachment_ids}, ensure_ascii=False, sort_keys=True)
|
||||
return store.create_message(str(chat_id), user['id'], body.text, body.attachment_ids, idempotency_key,
|
||||
hashlib.sha256(canonical.encode('utf-8')).digest(), settings.idempotency_ttl)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}/messages')
|
||||
def list_messages(chat_id: uuid.UUID, user: Annotated[dict, Depends(current_user)],
|
||||
limit: Limit = 50, cursor: Cursor = None):
|
||||
return store.list_messages(str(chat_id), user['id'], cursor, limit)
|
||||
|
||||
|
||||
uploads = threading.BoundedSemaphore(settings.upload_concurrency)
|
||||
|
||||
|
||||
def save_attachment(chat_id, owner_id, filename, content_type, spool, size, sha256):
|
||||
if not uploads.acquire(timeout=settings.upload_queue_seconds):
|
||||
raise ApiError(503, 'Слишком много одновременных загрузок, повторите позже')
|
||||
try:
|
||||
return store_upload(chat_id, owner_id, filename, content_type, spool, size, sha256)
|
||||
finally:
|
||||
uploads.release()
|
||||
|
||||
|
||||
def store_upload(chat_id, owner_id, filename, content_type, spool, size, sha256):
|
||||
images.inspect(spool, content_type)
|
||||
attachment_id = str(uuid.uuid4())
|
||||
object_key = 'attachments/' + attachment_id
|
||||
restart_budget()
|
||||
store.start_attachment(attachment_id, chat_id, owner_id, filename, content_type, size, sha256, object_key)
|
||||
restart_budget()
|
||||
try:
|
||||
s3.put(object_key, spool, size, content_type)
|
||||
except ApiError:
|
||||
try:
|
||||
store.forget_attachment(attachment_id)
|
||||
except psycopg.Error:
|
||||
pass
|
||||
raise
|
||||
if 'after_s3_put' in settings.faults and filename.startswith('fault-'):
|
||||
raise RuntimeError('fault injection: process failed after S3 PUT, before COMMIT of ready')
|
||||
restart_budget()
|
||||
return store.finish_attachment(attachment_id)
|
||||
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
|
||||
async def upload_attachment(chat_id: uuid.UUID, request: Request, user: Annotated[dict, Depends(current_user)],
|
||||
filename: Optional[str] = None):
|
||||
images.check_filename(filename)
|
||||
content_type = images.check_content_type(request.headers.get('content-type'))
|
||||
await run_in_threadpool(store.check_member, str(chat_id), user['id'])
|
||||
declared = request.headers.get('content-length', '')
|
||||
if declared.isdigit() and int(declared) > settings.max_upload_bytes:
|
||||
raise ApiError(413, 'Файл больше %d байт' % settings.max_upload_bytes)
|
||||
spool = tempfile.SpooledTemporaryFile(max_size=settings.upload_memory_bytes)
|
||||
try:
|
||||
size, digest = 0, hashlib.sha256()
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise ApiError(413, 'Файл больше %d байт' % settings.max_upload_bytes)
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
if size == 0:
|
||||
raise ApiError(415, 'Пустое тело вместо изображения')
|
||||
return await run_in_threadpool(save_attachment, str(chat_id), user['id'], filename, content_type,
|
||||
spool, size, digest.digest())
|
||||
finally:
|
||||
spool.close()
|
||||
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
def get_attachment(attachment_id: uuid.UUID, user: Annotated[dict, Depends(current_user)]):
|
||||
return store.attachment_out(store.get_attachment(str(attachment_id), user['id']))
|
||||
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def download_attachment(attachment_id: uuid.UUID, user: Annotated[dict, Depends(current_user)],
|
||||
variant: Literal['original'] = 'original'):
|
||||
row = store.get_attachment(str(attachment_id), user['id'])
|
||||
return StreamingResponse(s3.open_stream(row['object_key']), media_type=row['content_type'],
|
||||
headers={'Content-Length': str(row['size_bytes'])})
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import threading
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
import psycopg
|
||||
|
||||
from cache import REVOKED, cache
|
||||
from config import settings
|
||||
from db import iso
|
||||
from errors import BASIC_CHALLENGE, BEARER_CHALLENGE, unauthorized
|
||||
import store
|
||||
|
||||
REVOCATION_MARGIN_SECONDS = 5
|
||||
|
||||
USERNAME = re.compile(r'^[a-z0-9_]{3,32}$')
|
||||
|
||||
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
hashing = threading.BoundedSemaphore(settings.password_hash_concurrency)
|
||||
DUMMY_HASH = hasher.hash('dummy-password-for-equal-timing')
|
||||
|
||||
|
||||
def hash_password(password):
|
||||
with hashing:
|
||||
return hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash, password):
|
||||
try:
|
||||
with hashing:
|
||||
return hasher.verify(password_hash, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def token_hash(token):
|
||||
return hashlib.sha256(token.encode('ascii')).digest()
|
||||
|
||||
|
||||
def parse_credentials(header, scheme):
|
||||
name, _, value = (header or '').strip().partition(' ')
|
||||
value = value.strip()
|
||||
if name.lower() != scheme or not value or ' ' in value:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def parse_basic(header):
|
||||
value = parse_credentials(header, 'basic')
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(value, validate=True).decode('utf-8')
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
return None
|
||||
username, separator, password = decoded.partition(':')
|
||||
if not separator or not USERNAME.match(username) or not 12 <= len(password) <= 128:
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def login(header):
|
||||
credentials = parse_basic(header)
|
||||
row = store.find_credentials(credentials[0]) if credentials else None
|
||||
if row is None:
|
||||
verify_password(DUMMY_HASH, credentials[1] if credentials else 'x' * 12)
|
||||
raise unauthorized(BASIC_CHALLENGE, 'Неверное имя пользователя или пароль')
|
||||
if not verify_password(row['password_hash'], credentials[1]):
|
||||
raise unauthorized(BASIC_CHALLENGE, 'Неверное имя пользователя или пароль')
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = token_hash(token)
|
||||
expires_at = store.create_session(row['id'], digest, settings.session_ttl)
|
||||
user = store.user_out(row)
|
||||
cache.put(digest, user, expires_at)
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': iso(expires_at), 'user': user}
|
||||
|
||||
|
||||
def authenticate(header):
|
||||
token = parse_credentials(header, 'bearer')
|
||||
if token is None or not token.isascii():
|
||||
raise unauthorized(BEARER_CHALLENGE)
|
||||
digest = token_hash(token)
|
||||
if cache.client is not None and not cache.trusted():
|
||||
sync_revocations()
|
||||
user = cache.get(digest)
|
||||
if user == REVOKED:
|
||||
raise unauthorized(BEARER_CHALLENGE)
|
||||
if user is None:
|
||||
row = store.session_user(digest)
|
||||
if row is None:
|
||||
raise unauthorized(BEARER_CHALLENGE)
|
||||
user = store.user_out(row)
|
||||
cache.put(digest, user, row['expires_at'])
|
||||
return {'user': user, 'token_hash': digest}
|
||||
|
||||
|
||||
def sync_revocations():
|
||||
errors_before = cache.errors
|
||||
pending = store.pending_revocations()
|
||||
if not cache.revoke_all(pending):
|
||||
return
|
||||
if pending:
|
||||
try:
|
||||
store.clear_revocations(pending)
|
||||
except psycopg.Error:
|
||||
pass
|
||||
cache.mark_synced(errors_before)
|
||||
|
||||
|
||||
def logout(session):
|
||||
store.revoke_session(session['token_hash'], cache.max_ttl + REVOCATION_MARGIN_SECONDS)
|
||||
if cache.revoke(session['token_hash']):
|
||||
try:
|
||||
store.clear_revocations([session['token_hash']])
|
||||
except psycopg.Error:
|
||||
pass
|
||||
@@ -0,0 +1,89 @@
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
import redis
|
||||
|
||||
from config import settings
|
||||
|
||||
log = logging.getLogger('messenger.cache')
|
||||
|
||||
BYPASS_SECONDS = 5
|
||||
REVOKED = 'revoked'
|
||||
FAILED = object()
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url, max_ttl):
|
||||
self.client = redis.Redis.from_url(url, socket_timeout=0.2, socket_connect_timeout=0.2,
|
||||
decode_responses=True) if url else None
|
||||
self.max_ttl = max_ttl
|
||||
self.bypass_until = 0.0
|
||||
self.lock = threading.Lock()
|
||||
self.errors = 0
|
||||
self.synced = False
|
||||
self.synced_at = 0.0
|
||||
|
||||
def _run(self, action, force=False):
|
||||
if self.client is None:
|
||||
return None
|
||||
if not force and time.monotonic() < self.bypass_until:
|
||||
return FAILED
|
||||
try:
|
||||
return action(self.client)
|
||||
except redis.RedisError as error:
|
||||
log.warning('session cache unavailable (%s), using PostgreSQL for %ds', type(error).__name__, BYPASS_SECONDS)
|
||||
self.bypass_until = time.monotonic() + BYPASS_SECONDS
|
||||
with self.lock:
|
||||
self.errors += 1
|
||||
self.synced = False
|
||||
return FAILED
|
||||
|
||||
@staticmethod
|
||||
def key(token_hash):
|
||||
return 'session:' + token_hash.hex()
|
||||
|
||||
def mark_synced(self, errors_before):
|
||||
with self.lock:
|
||||
if self.errors == errors_before:
|
||||
self.synced = True
|
||||
self.synced_at = time.monotonic()
|
||||
|
||||
def trusted(self):
|
||||
return self.synced and time.monotonic() - self.synced_at < settings.revocation_sync_seconds
|
||||
|
||||
def get(self, token_hash):
|
||||
if not self.trusted():
|
||||
return None
|
||||
raw = self._run(lambda c: c.get(self.key(token_hash)))
|
||||
if raw is FAILED or not raw or raw == REVOKED:
|
||||
return REVOKED if raw == REVOKED else None
|
||||
entry = json.loads(raw)
|
||||
if entry['expires_at'] <= time.time():
|
||||
return None
|
||||
return entry['user']
|
||||
|
||||
def put(self, token_hash, user, expires_at: datetime):
|
||||
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
|
||||
ttl = min(self.max_ttl, remaining)
|
||||
if ttl <= 0:
|
||||
return
|
||||
value = json.dumps({'user': user, 'expires_at': expires_at.timestamp()}, ensure_ascii=False)
|
||||
self._run(lambda c: c.set(self.key(token_hash), value, ex=ttl, nx=True))
|
||||
|
||||
def revoke(self, token_hash):
|
||||
return self._run(lambda c: c.set(self.key(token_hash), REVOKED, ex=self.max_ttl), force=True) is not FAILED
|
||||
|
||||
def revoke_all(self, token_hashes):
|
||||
def write(client):
|
||||
with client.pipeline(transaction=False) as pipe:
|
||||
for token_hash in token_hashes:
|
||||
pipe.set(self.key(token_hash), REVOKED, ex=self.max_ttl)
|
||||
pipe.execute()
|
||||
return True
|
||||
return not token_hashes or self._run(write) is not FAILED
|
||||
|
||||
|
||||
cache = SessionCache(settings.cache_url, settings.session_cache_ttl)
|
||||
@@ -0,0 +1,54 @@
|
||||
import os
|
||||
|
||||
|
||||
def setting(name, default=None):
|
||||
path = os.environ.get(name + '_FILE')
|
||||
if path:
|
||||
with open(path, encoding='utf-8') as file:
|
||||
return file.read().strip()
|
||||
return os.environ.get(name, default)
|
||||
|
||||
|
||||
def positive_int(name, default):
|
||||
value = int(os.environ.get(name, default))
|
||||
if value <= 0:
|
||||
raise SystemExit('%s must be a positive integer' % name)
|
||||
return value
|
||||
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.host = os.environ.get('APP_HOST', '0.0.0.0')
|
||||
self.port = positive_int('APP_PORT', '8080')
|
||||
self.env = os.environ.get('APP_ENV', 'lab')
|
||||
self.shutdown_timeout = positive_int('APP_SHUTDOWN_TIMEOUT', '5')
|
||||
self.database_url = setting('DATABASE_URL')
|
||||
self.cache_url = setting('CACHE_URL') or None
|
||||
self.session_ttl = positive_int('SESSION_TTL_SECONDS', '3600')
|
||||
self.session_cache_ttl = positive_int('SESSION_CACHE_TTL_SECONDS', '60')
|
||||
self.db_pool_min = positive_int('DB_POOL_MIN', '1')
|
||||
self.db_pool_max = positive_int('DB_POOL_MAX', '10')
|
||||
self.db_pool_timeout = float(os.environ.get('DB_POOL_TIMEOUT_SECONDS', '2'))
|
||||
self.db_statement_timeout_ms = positive_int('DB_STATEMENT_TIMEOUT_MS', '3000')
|
||||
self.db_request_timeout = float(os.environ.get('DB_REQUEST_TIMEOUT_SECONDS', '4'))
|
||||
self.drain_seconds = float(os.environ.get('APP_DRAIN_SECONDS', '3'))
|
||||
self.idempotency_ttl = positive_int('IDEMPOTENCY_TTL_SECONDS', '86400')
|
||||
self.revocation_sync_seconds = float(os.environ.get('REVOCATION_SYNC_SECONDS', '1'))
|
||||
self.password_hash_concurrency = positive_int('PASSWORD_HASH_CONCURRENCY', '4')
|
||||
self.s3_endpoint = os.environ.get('S3_ENDPOINT', 'http://s3:8333')
|
||||
self.s3_region = os.environ.get('S3_REGION', 'us-east-1')
|
||||
self.s3_bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
self.s3_access_key = setting('S3_ACCESS_KEY_ID')
|
||||
self.s3_secret_key = setting('S3_SECRET_ACCESS_KEY')
|
||||
self.s3_timeout = float(os.environ.get('S3_TIMEOUT_SECONDS', '15'))
|
||||
self.max_upload_bytes = positive_int('MAX_UPLOAD_BYTES', '10485760')
|
||||
self.max_image_pixels = positive_int('MAX_IMAGE_PIXELS', '40000000')
|
||||
self.upload_memory_bytes = positive_int('UPLOAD_MEMORY_BYTES', '1048576')
|
||||
self.upload_concurrency = positive_int('UPLOAD_CONCURRENCY', '4')
|
||||
self.upload_queue_seconds = float(os.environ.get('UPLOAD_QUEUE_SECONDS', '30'))
|
||||
self.faults = set(filter(None, os.environ.get('FAULTS', '').split(',')))
|
||||
if not self.database_url:
|
||||
raise SystemExit('DATABASE_URL or DATABASE_URL_FILE is required')
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,133 @@
|
||||
from contextlib import contextmanager
|
||||
import contextvars
|
||||
from datetime import timezone
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from config import settings
|
||||
|
||||
log = logging.getLogger('messenger.db')
|
||||
|
||||
request_deadline = contextvars.ContextVar('request_deadline', default=None)
|
||||
|
||||
CHECK_SECONDS = 1
|
||||
|
||||
|
||||
def abort(conn):
|
||||
try:
|
||||
sock = socket.socket(fileno=os.dup(conn.fileno()))
|
||||
except (OSError, psycopg.Error):
|
||||
return
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
class Watchdog:
|
||||
def __init__(self):
|
||||
self.cond = threading.Condition()
|
||||
self.armed = {}
|
||||
self.ids = itertools.count()
|
||||
threading.Thread(target=self.loop, name='db-watchdog', daemon=True).start()
|
||||
|
||||
@contextmanager
|
||||
def guard(self, conn, seconds):
|
||||
key = next(self.ids)
|
||||
with self.cond:
|
||||
self.armed[key] = (time.monotonic() + seconds, conn)
|
||||
self.cond.notify()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self.cond:
|
||||
self.armed.pop(key, None)
|
||||
|
||||
def loop(self):
|
||||
with self.cond:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
for key, (deadline, conn) in list(self.armed.items()):
|
||||
if deadline <= now:
|
||||
del self.armed[key]
|
||||
log.warning('database did not answer in time, aborting connection')
|
||||
abort(conn)
|
||||
wake = min((deadline for deadline, _ in self.armed.values()), default=now + 60)
|
||||
self.cond.wait(max(wake - now, 0.01))
|
||||
|
||||
|
||||
watchdog = Watchdog()
|
||||
|
||||
|
||||
def check(conn):
|
||||
with watchdog.guard(conn, CHECK_SECONDS):
|
||||
ConnectionPool.check_connection(conn)
|
||||
|
||||
|
||||
pool = ConnectionPool(
|
||||
settings.database_url,
|
||||
min_size=settings.db_pool_min,
|
||||
max_size=settings.db_pool_max,
|
||||
timeout=settings.db_pool_timeout,
|
||||
check=check,
|
||||
open=False,
|
||||
name='messenger',
|
||||
kwargs={
|
||||
'connect_timeout': 3,
|
||||
'row_factory': dict_row,
|
||||
'options': '-c timezone=UTC -c statement_timeout=%d' % settings.db_statement_timeout_ms,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def restart_budget():
|
||||
budget = request_deadline.get()
|
||||
if budget is not None:
|
||||
budget['deadline'] = None
|
||||
|
||||
|
||||
def remaining(limit=None):
|
||||
budget = request_deadline.get()
|
||||
if budget is None:
|
||||
left = settings.db_request_timeout
|
||||
else:
|
||||
if budget['deadline'] is None:
|
||||
budget['deadline'] = time.monotonic() + settings.db_request_timeout
|
||||
left = budget['deadline'] - time.monotonic()
|
||||
if limit is not None:
|
||||
left = min(left, limit)
|
||||
if left <= 0:
|
||||
raise psycopg.OperationalError('request deadline exceeded')
|
||||
return left
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection(limit=None):
|
||||
conn = pool.getconn(timeout=min(settings.db_pool_timeout, remaining(limit)))
|
||||
try:
|
||||
with watchdog.guard(conn, remaining(limit)):
|
||||
try:
|
||||
yield conn
|
||||
except BaseException:
|
||||
try:
|
||||
conn.rollback()
|
||||
except psycopg.Error:
|
||||
pass
|
||||
raise
|
||||
conn.commit()
|
||||
finally:
|
||||
pool.putconn(conn)
|
||||
|
||||
|
||||
def iso(value):
|
||||
return value.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
|
||||
@@ -0,0 +1,37 @@
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
CODES = {
|
||||
400: 'invalid_request',
|
||||
401: 'unauthorized',
|
||||
404: 'not_found',
|
||||
409: 'conflict',
|
||||
413: 'payload_too_large',
|
||||
415: 'unsupported_media_type',
|
||||
429: 'rate_limited',
|
||||
500: 'internal_error',
|
||||
503: 'unavailable',
|
||||
}
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.headers = headers
|
||||
|
||||
|
||||
def unauthorized(challenge, message='Нужна действительная сессия'):
|
||||
return ApiError(401, message, {'WWW-Authenticate': challenge})
|
||||
|
||||
|
||||
def error_response(request: Request, status, message, headers=None):
|
||||
request_id = getattr(request.state, 'request_id', '-')
|
||||
body = {'error': {'code': CODES.get(status, 'invalid_request'), 'message': message, 'request_id': request_id}}
|
||||
if status == 401 and not headers:
|
||||
headers = {'WWW-Authenticate': BEARER_CHALLENGE}
|
||||
return JSONResponse(body, status_code=status, headers=headers)
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from config import settings
|
||||
from errors import ApiError
|
||||
|
||||
SIGNATURES = {'image/png': b'\x89PNG\r\n\x1a\n', 'image/jpeg': b'\xff\xd8\xff'}
|
||||
FORMATS = {'image/png': 'PNG', 'image/jpeg': 'JPEG'}
|
||||
FORBIDDEN_FILENAME = set('/\\\x7f') | {chr(code) for code in range(32)} | {chr(code) for code in range(0x80, 0xa0)}
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = settings.max_image_pixels
|
||||
|
||||
|
||||
def check_filename(filename):
|
||||
if filename is None or not 1 <= len(filename) <= 255 or not filename.strip() \
|
||||
or any(char in FORBIDDEN_FILENAME for char in filename):
|
||||
raise ApiError(400, 'Некорректный filename: 1–255 символов без /, \\ и управляющих символов')
|
||||
try:
|
||||
filename.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise ApiError(400, 'Некорректный filename') from None
|
||||
|
||||
|
||||
def check_content_type(header):
|
||||
content_type = (header or '').split(';')[0].strip().lower()
|
||||
if content_type not in SIGNATURES:
|
||||
raise ApiError(415, 'Принимаются только сырые байты image/png или image/jpeg')
|
||||
return content_type
|
||||
|
||||
|
||||
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
|
||||
|
||||
def png_rows(ihdr):
|
||||
width, height, depth, color, _, _, interlace = struct.unpack('>IIBBBBB', ihdr)
|
||||
bits = depth * CHANNELS[color]
|
||||
if interlace:
|
||||
passes = [(-(-(width - x0) // dx), -(-(height - y0) // dy)) for x0, y0, dx, dy in ADAM7]
|
||||
else:
|
||||
passes = [(width, height)]
|
||||
return [((w * bits + 7) // 8 + 1, h) for w, h in passes if w > 0 and h > 0]
|
||||
|
||||
|
||||
class PixelData:
|
||||
def __init__(self, ihdr, piece):
|
||||
self.rows = png_rows(ihdr)
|
||||
self.expected = sum(length * count for length, count in self.rows)
|
||||
self.inflater = zlib.decompressobj()
|
||||
self.piece = piece
|
||||
self.total = 0
|
||||
self.segment = 0
|
||||
self.offset = 0
|
||||
|
||||
def feed(self, data):
|
||||
while data:
|
||||
if self.inflater.eof:
|
||||
return
|
||||
out = self.inflater.decompress(data, self.piece)
|
||||
data = self.inflater.unconsumed_tail
|
||||
self.check(out)
|
||||
|
||||
def check(self, out):
|
||||
self.total += len(out)
|
||||
if self.total > self.expected:
|
||||
raise ValueError('PNG pixel data is longer than IHDR allows')
|
||||
while out:
|
||||
length, count = self.rows[self.segment]
|
||||
take = min(len(out), length * count - self.offset)
|
||||
part, out = out[:take], out[take:]
|
||||
filters = part[(-self.offset) % length::length]
|
||||
if filters and max(filters) > 4:
|
||||
raise ValueError('PNG row has an invalid filter type')
|
||||
self.offset += take
|
||||
if self.offset == length * count:
|
||||
self.segment, self.offset = self.segment + 1, 0
|
||||
|
||||
def finish(self):
|
||||
if not self.inflater.eof or self.total != self.expected:
|
||||
raise ValueError('PNG pixel data is truncated')
|
||||
|
||||
|
||||
def verify_png(fileobj, piece=64 * 1024):
|
||||
"""Потоково: CRC каждого чанка и распаковка IDAT кусками по piece байт без хранения пикселей."""
|
||||
fileobj.seek(8)
|
||||
pixels = None
|
||||
while True:
|
||||
header = fileobj.read(8)
|
||||
if len(header) < 8:
|
||||
raise ValueError('PNG ended before IEND')
|
||||
length, kind = struct.unpack('>I4s', header)
|
||||
if pixels is None and kind != b'IHDR':
|
||||
raise ValueError('PNG must start with IHDR')
|
||||
crc = zlib.crc32(kind)
|
||||
left = length
|
||||
while left:
|
||||
data = fileobj.read(min(piece, left))
|
||||
if not data:
|
||||
raise ValueError('PNG chunk is truncated')
|
||||
crc = zlib.crc32(data, crc)
|
||||
left -= len(data)
|
||||
if kind == b'IHDR':
|
||||
pixels = PixelData(data, piece)
|
||||
elif kind == b'IDAT':
|
||||
pixels.feed(data)
|
||||
stored = fileobj.read(4)
|
||||
if len(stored) < 4 or struct.unpack('>I', stored)[0] != crc & 0xffffffff:
|
||||
raise ValueError('PNG chunk CRC mismatch')
|
||||
if kind == b'IEND':
|
||||
pixels.finish()
|
||||
return
|
||||
|
||||
|
||||
def inspect(fileobj, content_type):
|
||||
fileobj.seek(0)
|
||||
if not fileobj.read(8).startswith(SIGNATURES[content_type]):
|
||||
raise ApiError(415, 'Содержимое файла не соответствует ' + content_type)
|
||||
fileobj.seek(0)
|
||||
try:
|
||||
with Image.open(fileobj, formats=[FORMATS[content_type]]) as image:
|
||||
width, height = image.size
|
||||
if width * height > settings.max_image_pixels:
|
||||
raise ApiError(413, 'Изображение больше %d пикселей' % settings.max_image_pixels)
|
||||
if content_type == 'image/png':
|
||||
verify_png(fileobj)
|
||||
else:
|
||||
image.draft('L', (max(1, width // 8), max(1, height // 8)))
|
||||
image.load()
|
||||
except ApiError:
|
||||
raise
|
||||
except Image.DecompressionBombError:
|
||||
raise ApiError(413, 'Изображение больше %d пикселей' % settings.max_image_pixels) from None
|
||||
except Exception:
|
||||
raise ApiError(415, 'Файл повреждён или не является изображением ' + content_type) from None
|
||||
finally:
|
||||
fileobj.seek(0)
|
||||
return width, height
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
import logging
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from config import settings
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
def handle_exit(self, sig, frame):
|
||||
from app import state
|
||||
if not state['ready'] or settings.drain_seconds <= 0:
|
||||
return super().handle_exit(sig, frame)
|
||||
state['ready'] = False
|
||||
logging.getLogger('messenger').info('signal %s: readiness off, draining for %.1fs', sig, settings.drain_seconds)
|
||||
threading.Timer(settings.drain_seconds, super().handle_exit, (sig, frame)).start()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
|
||||
format='%(asctime)s %(levelname)s %(name)s: %(message)s')
|
||||
logging.getLogger('messenger').info('config host=%s port=%d env=%s session_ttl=%ds session_cache_ttl=%ds',
|
||||
settings.host, settings.port, settings.env,
|
||||
settings.session_ttl, settings.session_cache_ttl)
|
||||
|
||||
from app import app
|
||||
config = uvicorn.Config(app, host=settings.host, port=settings.port, access_log=False, log_config=None,
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout)
|
||||
DrainingServer(config).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,47 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psycopg
|
||||
|
||||
from config import settings
|
||||
|
||||
MIGRATIONS = Path(__file__).resolve().parent.parent / 'migrations'
|
||||
LOCK_ID = 20260002
|
||||
|
||||
log = logging.getLogger('migrate')
|
||||
|
||||
|
||||
def connect(attempts=30):
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return psycopg.connect(settings.database_url, autocommit=True, connect_timeout=3)
|
||||
except psycopg.OperationalError:
|
||||
if attempt == attempts:
|
||||
raise
|
||||
log.info('database is not ready, retry %d/%d', attempt, attempts)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(name)s: %(message)s')
|
||||
with connect() as conn:
|
||||
conn.execute('SELECT pg_advisory_lock(%s)', (LOCK_ID,))
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations ('
|
||||
'version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())')
|
||||
applied = {row[0] for row in conn.execute('SELECT version FROM schema_migrations')}
|
||||
for path in sorted(MIGRATIONS.glob('*.sql')):
|
||||
if path.stem in applied:
|
||||
log.info('skip %s: already applied', path.stem)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(path.read_text(encoding='utf-8'))
|
||||
conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.stem,))
|
||||
log.info('applied %s', path.stem)
|
||||
conn.execute('SELECT pg_advisory_unlock(%s)', (LOCK_ID,))
|
||||
log.info('schema is up to date')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
import uuid
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid', strict=True)
|
||||
|
||||
|
||||
def not_blank(value):
|
||||
if not value.strip():
|
||||
raise ValueError('must not be blank')
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise ValueError('must be valid Unicode') from None
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('display_name', 'password')
|
||||
@classmethod
|
||||
def not_blank_fields(cls, value):
|
||||
return not_blank(value)
|
||||
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator('title')
|
||||
@classmethod
|
||||
def title_not_blank(cls, value):
|
||||
return not_blank(value)
|
||||
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
@classmethod
|
||||
def valid_attachment_ids(cls, value):
|
||||
try:
|
||||
ids = [str(uuid.UUID(item)) for item in value]
|
||||
except ValueError:
|
||||
raise ValueError('attachment_ids must be UUIDs') from None
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError('attachment_ids must be unique')
|
||||
return ids
|
||||
|
||||
@model_validator(mode='after')
|
||||
def text_or_attachments(self):
|
||||
if not self.attachment_ids:
|
||||
not_blank(self.text)
|
||||
else:
|
||||
self.text.encode('utf-8')
|
||||
return self
|
||||
@@ -0,0 +1,31 @@
|
||||
import base64
|
||||
from datetime import datetime
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from errors import ApiError
|
||||
|
||||
|
||||
def encode_cursor(scope, item):
|
||||
raw = json.dumps({'s': scope, 't': item['created_at'], 'i': item['id']}, separators=(',', ':'))
|
||||
return base64.urlsafe_b64encode(raw.encode()).decode().rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(scope, cursor):
|
||||
try:
|
||||
padded = cursor + '=' * (-len(cursor) % 4)
|
||||
data = json.loads(base64.urlsafe_b64decode(padded.encode('ascii')))
|
||||
if data['s'] != scope or not isinstance(data['t'], str) or not isinstance(data['i'], str):
|
||||
raise ValueError
|
||||
created_at = datetime.fromisoformat(data['t'])
|
||||
if created_at.tzinfo is None:
|
||||
raise ValueError
|
||||
return created_at, uuid.UUID(data['i'])
|
||||
except (ValueError, TypeError, KeyError, AttributeError):
|
||||
raise ApiError(400, 'Некорректный cursor') from None
|
||||
|
||||
|
||||
def page(items, scope, limit):
|
||||
chunk = items[:limit]
|
||||
has_more = len(items) > limit
|
||||
return {'items': chunk, 'next_cursor': encode_cursor(scope, chunk[-1]) if has_more else None}
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Сверка PostgreSQL (метаданные вложений) и S3 (байты).
|
||||
|
||||
python src/reconcile.py # dry-run: только отчёт
|
||||
python src/reconcile.py --apply # исправить то, что безопасно исправлять
|
||||
python src/reconcile.py --grace-seconds 900 --report /tmp/report.json
|
||||
python src/reconcile.py --apply --interval 600 # периодически (сервис reconciler в compose)
|
||||
|
||||
Категории:
|
||||
ok — ready-строка и объект на месте (размер совпадает);
|
||||
orphan — объект без строки, старше grace → удалить объект;
|
||||
fresh_orphan — объект без строки, моложе grace → не трогать (загрузка может идти прямо сейчас);
|
||||
stale_upload — строка 'uploading' старше grace → удалить строку и объект (если он есть);
|
||||
fresh_upload — строка 'uploading' моложе grace → не трогать;
|
||||
missing — ready-строка без объекта → только отчёт (восстановить нечего, удалять метаданные нельзя);
|
||||
size_mismatch — ready-строка и объект разного размера → только отчёт.
|
||||
Сначала читается список объектов, потом таблица: объект появляется в S3 только после строки 'uploading',
|
||||
поэтому у любого увиденного объекта, загруженного API, строка уже есть или была удалена компенсацией.
|
||||
"""
|
||||
import argparse
|
||||
from datetime import timezone
|
||||
import json
|
||||
import logging
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from config import setting, settings
|
||||
from s3 import make_client
|
||||
|
||||
PREFIX = 'attachments/'
|
||||
HEARTBEAT = '/tmp/reconciler.alive' # обновляется после каждого прохода цикла; по нему healthcheck
|
||||
|
||||
log = logging.getLogger('reconcile')
|
||||
|
||||
|
||||
def list_objects(client):
|
||||
objects, pages = {}, 0
|
||||
for page in client.get_paginator('list_objects_v2').paginate(Bucket=settings.s3_bucket, Prefix=PREFIX):
|
||||
pages += 1
|
||||
for item in page.get('Contents', []):
|
||||
objects[item['Key']] = {'size': item['Size'], 'modified': item['LastModified'].astimezone(timezone.utc)}
|
||||
return objects, pages
|
||||
|
||||
|
||||
def classify(objects, rows, now, grace):
|
||||
found = {name: [] for name in ('ok', 'orphan', 'fresh_orphan', 'stale_upload', 'fresh_upload', 'missing',
|
||||
'size_mismatch')}
|
||||
by_key = {row['object_key']: row for row in rows}
|
||||
for key, info in objects.items():
|
||||
if key not in by_key:
|
||||
age = (now - info['modified']).total_seconds()
|
||||
found['orphan' if age > grace else 'fresh_orphan'].append({'object_key': key, 'age_s': round(age)})
|
||||
for row in rows:
|
||||
age = (now - row['created_at']).total_seconds()
|
||||
item = {'id': str(row['id']), 'object_key': row['object_key'], 'age_s': round(age),
|
||||
'referenced': row['referenced'], 'object_exists': row['object_key'] in objects}
|
||||
if row['status'] == 'uploading':
|
||||
found['stale_upload' if age > grace else 'fresh_upload'].append(item)
|
||||
elif row['object_key'] not in objects:
|
||||
found['missing'].append(item)
|
||||
elif objects[row['object_key']]['size'] != row['size_bytes']:
|
||||
found['size_mismatch'].append(item)
|
||||
else:
|
||||
found['ok'].append(item)
|
||||
return found
|
||||
|
||||
|
||||
def apply(found, client, db):
|
||||
done = {'orphan_objects_deleted': 0, 'stale_rows_deleted': 0, 'stale_objects_deleted': 0, 'skipped': 0}
|
||||
for item in found['orphan']:
|
||||
if db.execute('SELECT 1 FROM attachments WHERE object_key = %s', (item['object_key'],)).fetchone():
|
||||
done['skipped'] += 1
|
||||
continue
|
||||
client.delete_object(Bucket=settings.s3_bucket, Key=item['object_key'])
|
||||
done['orphan_objects_deleted'] += 1
|
||||
for item in found['stale_upload']:
|
||||
deleted = db.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading' "
|
||||
"AND NOT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = %s) RETURNING 1",
|
||||
(item['id'], item['id'])).fetchone()
|
||||
db.commit()
|
||||
if not deleted:
|
||||
done['skipped'] += 1
|
||||
continue
|
||||
done['stale_rows_deleted'] += 1
|
||||
if item['object_exists']:
|
||||
client.delete_object(Bucket=settings.s3_bucket, Key=item['object_key'])
|
||||
done['stale_objects_deleted'] += 1
|
||||
return done
|
||||
|
||||
|
||||
def run_once(client, args):
|
||||
started = time.monotonic()
|
||||
with psycopg.connect(settings.database_url, row_factory=dict_row, connect_timeout=5) as db:
|
||||
objects, pages = list_objects(client)
|
||||
listed = time.monotonic()
|
||||
now = db.execute('SELECT now() AS now').fetchone()['now'].astimezone(timezone.utc)
|
||||
rows = db.execute('SELECT a.id, a.object_key, a.status, a.size_bytes, a.created_at, '
|
||||
'EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) AS referenced '
|
||||
'FROM attachments a').fetchall()
|
||||
db.commit()
|
||||
scanned = time.monotonic()
|
||||
found = classify(objects, rows, now, args.grace_seconds)
|
||||
done = apply(found, client, db) if args.apply else None
|
||||
report = {
|
||||
'mode': 'apply' if args.apply else 'dry-run', 'grace_seconds': args.grace_seconds,
|
||||
'objects_listed': len(objects), 'list_pages': pages, 'rows_scanned': len(rows),
|
||||
'counts': {name: len(items) for name, items in found.items()},
|
||||
'actions': done,
|
||||
'seconds': {'list_s3': round(listed - started, 3), 'scan_db': round(scanned - listed, 3),
|
||||
'total': round(time.monotonic() - started, 3)},
|
||||
'max_rss_mib': round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1),
|
||||
'missing': found['missing'], 'size_mismatch': found['size_mismatch'],
|
||||
}
|
||||
if args.details:
|
||||
report['items'] = {name: items for name, items in found.items() if name != 'ok'}
|
||||
return report
|
||||
|
||||
|
||||
def run_forever(client, args):
|
||||
"""Периодический режим (сервис reconciler): прогон сразу и далее каждые --interval секунд."""
|
||||
stop = threading.Event()
|
||||
signal.signal(signal.SIGTERM, lambda *_: stop.set())
|
||||
log.info('periodic mode: every %.0fs, %s, grace %.0fs', args.interval,
|
||||
'apply' if args.apply else 'dry-run', args.grace_seconds)
|
||||
while not stop.is_set():
|
||||
try:
|
||||
report = run_once(client, args)
|
||||
log.info('run: objects %d, rows %d, counts %s, actions %s, %.2fs', report['objects_listed'],
|
||||
report['rows_scanned'], {k: v for k, v in report['counts'].items() if v},
|
||||
report['actions'], report['seconds']['total'])
|
||||
except Exception as error:
|
||||
log.warning('run failed, will retry next time: %s: %s', type(error).__name__, error)
|
||||
with open(HEARTBEAT, 'w') as file:
|
||||
file.write(str(time.time()))
|
||||
stop.wait(args.interval)
|
||||
log.info('stopped')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--grace-seconds', type=float, default=900)
|
||||
parser.add_argument('--interval', type=float, default=0, help='repeat every N seconds (0 = run once)')
|
||||
parser.add_argument('--report')
|
||||
parser.add_argument('--details', action='store_true', help='include every item in the report')
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(asctime)s %(name)s: %(message)s')
|
||||
client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30)
|
||||
if args.interval > 0:
|
||||
run_forever(client, args)
|
||||
return
|
||||
text = json.dumps(run_once(client, args), ensure_ascii=False, indent=2, default=str)
|
||||
if args.report:
|
||||
with open(args.report, 'w', encoding='utf-8') as file:
|
||||
file.write(text)
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
import logging
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from config import settings
|
||||
from errors import ApiError
|
||||
|
||||
log = logging.getLogger('messenger.s3')
|
||||
|
||||
CHUNK = 64 * 1024
|
||||
|
||||
|
||||
def make_client(access_key, secret_key, timeout=None):
|
||||
return boto3.client(
|
||||
's3', endpoint_url=settings.s3_endpoint, region_name=settings.s3_region,
|
||||
aws_access_key_id=access_key, aws_secret_access_key=secret_key,
|
||||
config=Config(signature_version='s3v4', s3={'addressing_style': 'path'},
|
||||
connect_timeout=timeout or settings.s3_timeout, read_timeout=timeout or settings.s3_timeout,
|
||||
retries={'max_attempts': 2, 'mode': 'standard'},
|
||||
request_checksum_calculation='when_required', response_checksum_validation='when_required'))
|
||||
|
||||
|
||||
client = make_client(settings.s3_access_key, settings.s3_secret_key) if settings.s3_access_key else None
|
||||
|
||||
|
||||
def unavailable(error):
|
||||
log.warning('object storage unavailable: %s', type(error).__name__)
|
||||
return ApiError(503, 'Хранилище файлов временно недоступно')
|
||||
|
||||
|
||||
def put(key, fileobj, size, content_type):
|
||||
try:
|
||||
client.put_object(Bucket=settings.s3_bucket, Key=key, Body=fileobj, ContentLength=size,
|
||||
ContentType=content_type)
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
raise unavailable(error) from None
|
||||
|
||||
|
||||
def delete(key):
|
||||
try:
|
||||
client.delete_object(Bucket=settings.s3_bucket, Key=key)
|
||||
return True
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
log.warning('could not delete object %s: %s', key, type(error).__name__)
|
||||
return False
|
||||
|
||||
|
||||
def open_stream(key):
|
||||
try:
|
||||
body = client.get_object(Bucket=settings.s3_bucket, Key=key)['Body']
|
||||
except ClientError as error:
|
||||
if error.response.get('Error', {}).get('Code') in ('NoSuchKey', '404'):
|
||||
log.error('object %s is missing for a ready attachment', key)
|
||||
raise unavailable(error) from None
|
||||
except BotoCoreError as error:
|
||||
raise unavailable(error) from None
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
yield from body.iter_chunks(CHUNK)
|
||||
finally:
|
||||
body.close()
|
||||
return chunks()
|
||||
@@ -0,0 +1,55 @@
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from config import setting, settings
|
||||
from s3 import make_client
|
||||
|
||||
log = logging.getLogger('s3-bootstrap')
|
||||
|
||||
|
||||
def wait_for(client, attempts=30):
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return client.list_buckets()
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
if attempt == attempts:
|
||||
raise
|
||||
log.info('object storage is not ready (%s), retry %d/%d', type(error).__name__, attempt, attempts)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def anonymous_status(path):
|
||||
try:
|
||||
urllib.request.urlopen(settings.s3_endpoint + path, timeout=5)
|
||||
return 200
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(name)s: %(message)s')
|
||||
admin = make_client(setting('S3_ADMIN_ACCESS_KEY_ID'), setting('S3_ADMIN_SECRET_ACCESS_KEY'))
|
||||
buckets = {bucket['Name'] for bucket in wait_for(admin).get('Buckets', [])}
|
||||
if settings.s3_bucket in buckets:
|
||||
log.info('bucket %s already exists, nothing to do', settings.s3_bucket)
|
||||
else:
|
||||
admin.create_bucket(Bucket=settings.s3_bucket)
|
||||
log.info('bucket %s created', settings.s3_bucket)
|
||||
started = time.monotonic()
|
||||
admin.put_object(Bucket=settings.s3_bucket, Key='bootstrap/warmup', Body=b'warmup')
|
||||
admin.delete_object(Bucket=settings.s3_bucket, Key='bootstrap/warmup')
|
||||
log.info('warm-up write done in %.1fs (storage volumes allocated before first upload)', time.monotonic() - started)
|
||||
for path in ('/' + settings.s3_bucket, '/' + settings.s3_bucket + '/probe'):
|
||||
status = anonymous_status(path)
|
||||
if status not in (401, 403):
|
||||
raise SystemExit('anonymous access to %s returned %d, bucket is not private' % (path, status))
|
||||
log.info('anonymous list/get denied: bucket %s is private', settings.s3_bucket)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
import uuid
|
||||
|
||||
from psycopg import errors
|
||||
|
||||
from db import connection, iso
|
||||
from errors import ApiError
|
||||
from pagination import decode_cursor, page
|
||||
|
||||
USER_COLUMNS = 'u.id, u.username, u.display_name, u.created_at'
|
||||
ATTACHMENT_IDS = ('ARRAY(SELECT ma.attachment_id FROM message_attachments ma '
|
||||
'WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids')
|
||||
MESSAGE_COLUMNS = 'm.id, m.chat_id, m.sender_id, m.text, m.created_at, ' + ATTACHMENT_IDS
|
||||
ATTACHMENT_COLUMNS = 'a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, a.status, a.created_at, a.updated_at'
|
||||
CHAT_SELECT = ('SELECT c.id, c.title, c.created_at, '
|
||||
'ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids '
|
||||
'FROM chats c ')
|
||||
|
||||
|
||||
def utf8(value):
|
||||
return value.encode('utf-8')
|
||||
|
||||
|
||||
def unicode(value):
|
||||
return bytes(value).decode('utf-8')
|
||||
|
||||
|
||||
def user_out(row):
|
||||
return {'id': str(row['id']), 'username': row['username'],
|
||||
'display_name': unicode(row['display_name']), 'created_at': iso(row['created_at'])}
|
||||
|
||||
|
||||
def chat_out(row):
|
||||
return {'id': str(row['id']), 'title': unicode(row['title']),
|
||||
'member_ids': [str(m) for m in row['member_ids']], 'created_at': iso(row['created_at'])}
|
||||
|
||||
|
||||
def message_out(row):
|
||||
return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'sender_id': str(row['sender_id']),
|
||||
'text': unicode(row['text']), 'created_at': iso(row['created_at']),
|
||||
'attachment_ids': [str(a) for a in row['attachment_ids']]}
|
||||
|
||||
|
||||
def attachment_out(row):
|
||||
return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']),
|
||||
'filename': row['filename'], 'content_type': row['content_type'], 'size_bytes': row['size_bytes'],
|
||||
'sha256': bytes(row['sha256']).hex(), 'status': row['status'], 'variants': [], 'error_code': None,
|
||||
'created_at': iso(row['created_at']), 'updated_at': iso(row['updated_at'])}
|
||||
|
||||
|
||||
def create_user(username, display_name, password_hash):
|
||||
try:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
'INSERT INTO users AS u (username, display_name, password_hash) VALUES (%s, %s, %s) '
|
||||
'RETURNING ' + USER_COLUMNS,
|
||||
(username, utf8(display_name), password_hash)).fetchone()
|
||||
except errors.UniqueViolation:
|
||||
raise ApiError(409, 'Имя пользователя уже занято') from None
|
||||
return user_out(row)
|
||||
|
||||
|
||||
def find_credentials(username):
|
||||
with connection() as conn:
|
||||
return conn.execute('SELECT ' + USER_COLUMNS + ', u.password_hash FROM users u WHERE u.username = %s',
|
||||
(username,)).fetchone()
|
||||
|
||||
|
||||
def create_session(user_id, token_hash, ttl_seconds):
|
||||
with connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE user_id = %s AND expires_at <= now()', (user_id,))
|
||||
return conn.execute(
|
||||
'INSERT INTO sessions (token_hash, user_id, expires_at) '
|
||||
'VALUES (%s, %s, now() + make_interval(secs => %s)) RETURNING expires_at',
|
||||
(token_hash, user_id, ttl_seconds)).fetchone()['expires_at']
|
||||
|
||||
|
||||
def session_user(token_hash):
|
||||
with connection() as conn:
|
||||
return conn.execute(
|
||||
'SELECT ' + USER_COLUMNS + ', s.expires_at FROM sessions s JOIN users u ON u.id = s.user_id '
|
||||
'WHERE s.token_hash = %s AND s.expires_at > now()',
|
||||
(token_hash,)).fetchone()
|
||||
|
||||
|
||||
def revoke_session(token_hash, keep_seconds):
|
||||
with connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
|
||||
conn.execute('INSERT INTO session_revocations (token_hash, expires_at) '
|
||||
'VALUES (%s, now() + make_interval(secs => %s)) '
|
||||
'ON CONFLICT (token_hash) DO UPDATE SET expires_at = EXCLUDED.expires_at',
|
||||
(token_hash, keep_seconds))
|
||||
|
||||
|
||||
def pending_revocations():
|
||||
with connection() as conn:
|
||||
conn.execute('DELETE FROM session_revocations WHERE expires_at <= now()')
|
||||
return [bytes(r['token_hash']) for r in conn.execute('SELECT token_hash FROM session_revocations').fetchall()]
|
||||
|
||||
|
||||
def clear_revocations(token_hashes):
|
||||
with connection() as conn:
|
||||
conn.execute('DELETE FROM session_revocations WHERE token_hash = ANY(%s)', (list(token_hashes),))
|
||||
|
||||
|
||||
def create_chat(creator_id, title, member_ids):
|
||||
try:
|
||||
members = [str(uuid.UUID(m)) for m in member_ids]
|
||||
except ValueError:
|
||||
raise ApiError(400, 'Неизвестный участник чата') from None
|
||||
if len(set(members)) != len(members):
|
||||
raise ApiError(400, 'Участники не должны повторяться')
|
||||
if creator_id not in members:
|
||||
raise ApiError(400, 'Создатель должен быть участником чата')
|
||||
try:
|
||||
with connection() as conn, conn.transaction():
|
||||
chat = conn.execute('INSERT INTO chats (title, created_by) VALUES (%s, %s) RETURNING id, title, created_at',
|
||||
(utf8(title), creator_id)).fetchone()
|
||||
conn.execute('INSERT INTO chat_members (chat_id, user_id, position) '
|
||||
'SELECT %s, member, position FROM unnest(%s::uuid[]) WITH ORDINALITY AS t(member, position)',
|
||||
(chat['id'], members))
|
||||
except errors.ForeignKeyViolation:
|
||||
raise ApiError(400, 'Неизвестный участник чата') from None
|
||||
return chat_out({**chat, 'member_ids': members})
|
||||
|
||||
|
||||
def get_chat(chat_id, user_id):
|
||||
with connection() as conn:
|
||||
row = conn.execute(CHAT_SELECT + 'JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s '
|
||||
'WHERE c.id = %s', (user_id, chat_id)).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(404, 'Чат не найден')
|
||||
return chat_out(row)
|
||||
|
||||
|
||||
def list_chats(user_id, cursor, limit):
|
||||
scope = 'user:' + user_id
|
||||
sql = CHAT_SELECT + 'JOIN chat_members me ON me.chat_id = c.id WHERE me.user_id = %s '
|
||||
params = [user_id]
|
||||
if cursor:
|
||||
sql += 'AND (c.created_at, c.id) > (%s, %s) '
|
||||
params += decode_cursor(scope, cursor)
|
||||
sql += 'ORDER BY c.created_at, c.id LIMIT %s'
|
||||
with connection() as conn:
|
||||
rows = conn.execute(sql, params + [limit + 1]).fetchall()
|
||||
return page([chat_out(r) for r in rows], scope, limit)
|
||||
|
||||
|
||||
def is_member(conn, chat_id, user_id):
|
||||
return conn.execute('SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s',
|
||||
(chat_id, user_id)).fetchone() is not None
|
||||
|
||||
|
||||
def create_message(chat_id, sender_id, text, attachment_ids, key, request_hash, key_ttl_seconds):
|
||||
scope = 'POST /api/v1/chats/%s/messages' % chat_id
|
||||
with connection() as conn, conn.transaction():
|
||||
conn.execute('DELETE FROM idempotency_keys WHERE user_id = %s AND created_at < now() - make_interval(secs => %s)',
|
||||
(sender_id, key_ttl_seconds))
|
||||
claimed = conn.execute(
|
||||
'INSERT INTO idempotency_keys (user_id, scope, key, request_hash) VALUES (%s, %s, %s, %s) '
|
||||
'ON CONFLICT DO NOTHING RETURNING 1',
|
||||
(sender_id, scope, key, request_hash)).fetchone()
|
||||
if claimed is None:
|
||||
if not is_member(conn, chat_id, sender_id):
|
||||
raise ApiError(404, 'Чат не найден')
|
||||
stored = conn.execute(
|
||||
'SELECT k.request_hash, ' + MESSAGE_COLUMNS + ' '
|
||||
'FROM idempotency_keys k JOIN messages m ON m.id = k.message_id '
|
||||
'WHERE k.user_id = %s AND k.scope = %s AND k.key = %s',
|
||||
(sender_id, scope, key)).fetchone()
|
||||
if bytes(stored['request_hash']) != request_hash:
|
||||
raise ApiError(409, 'Idempotency-Key уже использован с другим запросом')
|
||||
return message_out(stored)
|
||||
row = conn.execute(
|
||||
'INSERT INTO messages AS m (chat_id, sender_id, text) SELECT %s, %s, %s '
|
||||
'WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s) '
|
||||
'RETURNING m.id, m.chat_id, m.sender_id, m.text, m.created_at',
|
||||
(chat_id, sender_id, utf8(text), chat_id, sender_id)).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(404, 'Чат не найден')
|
||||
if attachment_ids:
|
||||
found = conn.execute("SELECT count(*) AS n FROM attachments WHERE id = ANY(%s::uuid[]) "
|
||||
"AND chat_id = %s AND status = 'ready'", (attachment_ids, chat_id)).fetchone()['n']
|
||||
if found != len(attachment_ids):
|
||||
raise ApiError(404, 'Вложение не найдено в этом чате')
|
||||
conn.execute('INSERT INTO message_attachments (message_id, attachment_id, position) '
|
||||
'SELECT %s, attachment, position FROM unnest(%s::uuid[]) WITH ORDINALITY AS t(attachment, position)',
|
||||
(row['id'], attachment_ids))
|
||||
row = dict(row, attachment_ids=attachment_ids)
|
||||
conn.execute('UPDATE idempotency_keys SET message_id = %s WHERE user_id = %s AND scope = %s AND key = %s',
|
||||
(row['id'], sender_id, scope, key))
|
||||
return message_out(row)
|
||||
|
||||
|
||||
def list_messages(chat_id, user_id, cursor, limit):
|
||||
scope = 'chat:' + chat_id
|
||||
sql = 'SELECT ' + MESSAGE_COLUMNS + ' FROM messages m WHERE m.chat_id = %s '
|
||||
params = [chat_id]
|
||||
if cursor:
|
||||
sql += 'AND (m.created_at, m.id) > (%s, %s) '
|
||||
params += decode_cursor(scope, cursor)
|
||||
sql += 'ORDER BY m.created_at, m.id LIMIT %s'
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'Чат не найден')
|
||||
rows = conn.execute(sql, params + [limit + 1]).fetchall()
|
||||
return page([message_out(r) for r in rows], scope, limit)
|
||||
|
||||
|
||||
def start_attachment(attachment_id, chat_id, owner_id, filename, content_type, size, sha256, object_key):
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, owner_id):
|
||||
raise ApiError(404, 'Чат не найден')
|
||||
conn.execute("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading')",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size, sha256, object_key))
|
||||
|
||||
|
||||
def finish_attachment(attachment_id):
|
||||
with connection() as conn:
|
||||
row = conn.execute("UPDATE attachments AS a SET status = 'ready', updated_at = clock_timestamp() "
|
||||
"WHERE a.id = %s AND a.status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS,
|
||||
(attachment_id,)).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(503, 'Загрузка не завершена, повторите')
|
||||
return attachment_out(row)
|
||||
|
||||
|
||||
def forget_attachment(attachment_id):
|
||||
with connection() as conn:
|
||||
conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
|
||||
|
||||
|
||||
def check_member(chat_id, user_id):
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise ApiError(404, 'Чат не найден')
|
||||
|
||||
|
||||
def get_attachment(attachment_id, user_id):
|
||||
with connection() as conn:
|
||||
row = conn.execute("SELECT " + ATTACHMENT_COLUMNS + " FROM attachments a "
|
||||
"JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s "
|
||||
"WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id)).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(404, 'Вложение не найдено')
|
||||
return row
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Usage (стенд запущен через docker compose up -d):
|
||||
python student-tests/consistency_check.py
|
||||
|
||||
Проверяет, что сбои при загрузке не оставляют вечных неучтённых данных:
|
||||
1) клиент обрывает загрузку посреди тела → ни строки, ни объекта;
|
||||
2) S3 остановлен → upload/download 503, ни строки, ни объекта; метаданные, текст и ready работают;
|
||||
3) процесс «падает» после S3 PUT до COMMIT ready (FAULTS=after_s3_put, только в этом тесте) →
|
||||
остаётся строка uploading + объект; сверщик не трогает их в пределах grace и убирает после;
|
||||
4) объект без строки в БД (orphan) → свежий не трогается, старый удаляется; повторный запуск ничего не делает;
|
||||
5) периодический сервис reconciler сам удаляет заготовку старше grace (без ручного запуска).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
from helpers import BASE_URL, call, compose, new_chat, new_user, sql, wait_ready
|
||||
from imagegen import SAMPLE_PNG
|
||||
from test_attachments import download, upload
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(title, ok, detail=''):
|
||||
print(' %-62s %s %s' % (title, 'OK' if ok else 'FAIL', detail))
|
||||
if not ok:
|
||||
failures.append(title)
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
return json.loads(compose('run', '--rm', 'reconcile', *args))
|
||||
|
||||
|
||||
def s3_python(code):
|
||||
return compose('run', '--rm', '--entrypoint', 'python', 'reconcile', '-c',
|
||||
'import sys; sys.path.insert(0, "src")\n' + code)
|
||||
|
||||
|
||||
def counts():
|
||||
return sql("SELECT count(*) FILTER (WHERE status = 'ready') || '/' || count(*) FILTER (WHERE status = 'uploading') "
|
||||
"FROM attachments")
|
||||
|
||||
|
||||
def object_exists(key):
|
||||
return s3_python('from config import setting, settings\nfrom s3 import make_client\n'
|
||||
'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n'
|
||||
'keys = [o["Key"] for o in c.list_objects_v2(Bucket=settings.s3_bucket, Prefix=%r).get("Contents", [])]\n'
|
||||
'print(len(keys))' % key).strip() == '1'
|
||||
|
||||
|
||||
def restart_api(faults=None):
|
||||
env = dict(os.environ)
|
||||
env['FAULTS'] = faults or ''
|
||||
compose('up', '-d', '--no-deps', '--no-build', 'api', env=env)
|
||||
time.sleep(3)
|
||||
wait_ready()
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def interrupted_upload(alice, chat):
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
with socket.create_connection((target.hostname, target.port), timeout=10) as sock:
|
||||
head = ('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n'
|
||||
'Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n'
|
||||
% (chat['id'], alice['token'], 5 * 1024 * 1024))
|
||||
sock.sendall(head.encode() + SAMPLE_PNG + bytes(512 * 1024))
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def main():
|
||||
alice = new_user('consistency')
|
||||
chat = new_chat(alice)
|
||||
|
||||
print('1. Обрыв загрузки посреди тела')
|
||||
before = counts()
|
||||
interrupted_upload(alice, chat)
|
||||
check('строк не прибавилось (ready/uploading)', counts() == before, '%s -> %s' % (before, counts()))
|
||||
|
||||
print('2. S3 остановлен: загрузка, скачивание, остальной API')
|
||||
stored = upload(alice, chat, SAMPLE_PNG)[2]
|
||||
before = counts()
|
||||
compose('stop', 's3')
|
||||
try:
|
||||
started = time.monotonic()
|
||||
status = upload(alice, chat, SAMPLE_PNG)[0]
|
||||
check('upload при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started))
|
||||
check('заготовка удалена компенсацией', counts() == before, '%s -> %s' % (before, counts()))
|
||||
started = time.monotonic()
|
||||
status = download(alice, stored['id'])[0]
|
||||
check('download при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started))
|
||||
check('метаданные вложения доступны (они в PostgreSQL)',
|
||||
call('GET', '/api/v1/attachments/' + stored['id'], token=alice['token'])[0] == 200)
|
||||
check('текстовое сообщение → 201', call('POST', '/api/v1/chats/%s/messages' % chat['id'],
|
||||
{'text': 'без S3'}, alice['token'])[0] == 201)
|
||||
check('/health/ready → 200 (S3 не критичен для текстового API)', call('GET', '/health/ready')[0] == 200)
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 's3')
|
||||
check('после старта S3 upload снова 201', upload(alice, chat, SAMPLE_PNG)[0] == 201)
|
||||
|
||||
print('3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put)')
|
||||
restart_api('after_s3_put')
|
||||
try:
|
||||
status = upload(alice, chat, SAMPLE_PNG, filename='fault-%s.png' % uuid.uuid4().hex[:6])[0]
|
||||
row = sql("SELECT id || ' ' || object_key FROM attachments WHERE status = 'uploading' "
|
||||
"AND filename LIKE 'fault-%%' ORDER BY created_at DESC LIMIT 1").split()
|
||||
check('клиент получил ошибку (не 201)', status != 201, str(status))
|
||||
check('осталась строка uploading и объект в S3', bool(row) and object_exists(row[1]), ' '.join(row))
|
||||
finally:
|
||||
restart_api()
|
||||
fresh = reconcile('--grace-seconds', '900')
|
||||
check('dry-run, grace 900 с: заготовка свежая, не трогается', fresh['counts']['fresh_upload'] >= 1
|
||||
and fresh['counts']['stale_upload'] == 0, str(fresh['counts']))
|
||||
applied = reconcile('--apply', '--grace-seconds', '0')
|
||||
check('apply, grace 0: строка и объект удалены', applied['actions']['stale_rows_deleted'] >= 1
|
||||
and applied['actions']['stale_objects_deleted'] >= 1, str(applied['actions']))
|
||||
check('объекта больше нет', not object_exists(row[1]))
|
||||
|
||||
print('4. Объект без строки в БД (orphan)')
|
||||
key = 'attachments/%s' % uuid.uuid4()
|
||||
s3_python('from config import setting, settings\nfrom s3 import make_client\n'
|
||||
'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n'
|
||||
'c.put_object(Bucket=settings.s3_bucket, Key=%r, Body=b"orphan")' % key)
|
||||
fresh = reconcile('--grace-seconds', '900')
|
||||
check('dry-run, grace 900 с: свежая сирота не трогается', fresh['counts']['fresh_orphan'] >= 1
|
||||
and fresh['counts']['orphan'] == 0, str(fresh['counts']))
|
||||
applied = reconcile('--apply', '--grace-seconds', '0')
|
||||
check('apply, grace 0: сирота удалена', applied['actions']['orphan_objects_deleted'] >= 1, str(applied['actions']))
|
||||
again = reconcile('--apply', '--grace-seconds', '0')
|
||||
check('повторный запуск ничего не делает', not any(again['actions'][k] for k in
|
||||
('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), str(again['actions']))
|
||||
check('рабочие вложения не пострадали (missing = 0)', again['counts']['missing'] == 0, str(again['counts']))
|
||||
|
||||
print('5. Периодический сервис reconciler убирает зависшую заготовку сам')
|
||||
stale = str(uuid.uuid4())
|
||||
sql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, "
|
||||
"status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'stale.png', 'image/png', 1, "
|
||||
"decode(repeat('00', 32), 'hex'), 'attachments/%s', 'uploading', now() - interval '1 hour', "
|
||||
"now() - interval '1 hour')" % (stale, chat['id'], alice['id'], stale))
|
||||
running = 'reconciler' in compose('ps', '--status', 'running', '--services')
|
||||
check('сервис reconciler запущен (docker compose ps)', running)
|
||||
if running:
|
||||
compose('restart', 'reconciler') # проход выполняется сразу после старта, дальше — раз в интервал
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline and sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) != '0':
|
||||
time.sleep(1)
|
||||
check('заготовка старше grace удалена без ручного запуска',
|
||||
sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) == '0')
|
||||
|
||||
print('FAIL: ' + ', '.join(failures) if failures else 'PASS: сбои загрузки не оставляют неучтённых данных')
|
||||
raise SystemExit(1 if failures else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Usage (стенд уже запущен через docker compose up -d):
|
||||
python student-tests/expiry_check.py --ttl 2
|
||||
|
||||
Скрипт пересоздаёт только api с SESSION_TTL_SECONDS=<ttl>, проверяет истечение сессии,
|
||||
затем возвращает api с обычным TTL из .env. Volumes и данные не трогаются.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
|
||||
from helpers import call, compose, new_user, wait_ready
|
||||
|
||||
|
||||
def restart_api(ttl=None):
|
||||
env = dict(os.environ)
|
||||
if ttl is None:
|
||||
env.pop('SESSION_TTL_SECONDS', None)
|
||||
else:
|
||||
env['SESSION_TTL_SECONDS'] = str(ttl)
|
||||
compose('up', '-d', '--no-deps', '--no-build', 'api', env=env)
|
||||
wait_ready()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--ttl', type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
|
||||
print('пересоздаю api с SESSION_TTL_SECONDS=%d' % args.ttl)
|
||||
restart_api(args.ttl)
|
||||
try:
|
||||
alice = new_user('expiry')
|
||||
status = call('GET', '/api/v1/users/me', token=alice['token'])[0]
|
||||
print('сразу после входа: /users/me -> %d' % status)
|
||||
assert status == 200
|
||||
key = 'session:' + hashlib.sha256(alice['token'].encode()).hexdigest()
|
||||
cache_ttl = int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key))
|
||||
print('TTL ключа этой сессии в Valkey: %d с (-2 = ключа нет)' % cache_ttl)
|
||||
assert cache_ttl <= args.ttl, 'кэш живёт дольше сессии'
|
||||
|
||||
time.sleep(args.ttl + 1.5)
|
||||
status, headers, _ = call('GET', '/api/v1/users/me', token=alice['token'])
|
||||
print('через %.1f с: /users/me -> %d, WWW-Authenticate: %s' % (args.ttl + 1.5, status, headers['WWW-Authenticate']))
|
||||
assert status == 401 and headers['WWW-Authenticate'] == 'Bearer realm="messenger"'
|
||||
status = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0]
|
||||
print('logout истёкшим токеном -> %d' % status)
|
||||
assert status == 401
|
||||
print('PASS: сессия истекла по SESSION_TTL_SECONDS=%d, кэш не продлил её' % args.ttl)
|
||||
finally:
|
||||
print('возвращаю api с TTL из .env')
|
||||
restart_api()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Usage (стенд запущен через docker compose up -d, обе реплики в ротации):
|
||||
python student-tests/failover_series.py --output evidence/failover-series.json
|
||||
|
||||
Для каждой API-реплики по очереди и в двух режимах (docker stop — штатная остановка с выводом из ротации,
|
||||
docker kill — внезапная смерть процесса) под нагрузкой: останавливает реплику, меряет ошибки, RPS,
|
||||
время до последнего ответа остановленной реплики, восстанавливает её (docker compose start api)
|
||||
и ждёт возврата в ротацию LB. Volumes, БД и LB не трогаются.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from helpers import Load, api_containers, compose, new_chat, new_user, wait_in_rotation
|
||||
|
||||
WARMUP, AFTER = 5, 10
|
||||
|
||||
|
||||
def run(container, mode):
|
||||
alice = new_user('series')
|
||||
chat = new_chat(alice)
|
||||
load = Load(alice['token'], chat['id']).start()
|
||||
time.sleep(WARMUP)
|
||||
acted = time.monotonic()
|
||||
command = ['docker', 'stop', '-t', '15', container] if mode == 'stop' else ['docker', 'kill', container]
|
||||
subprocess.run(command, capture_output=True, check=True)
|
||||
action_seconds = time.monotonic() - acted
|
||||
time.sleep(AFTER)
|
||||
load.stop()
|
||||
instance = container[:12]
|
||||
target = [r for r in load.records if r[2] == 201 and r[3] == instance and r[0] >= acted]
|
||||
others = [r for r in load.records if r[2] == 201 and r[3] != instance and r[0] >= acted]
|
||||
restored = time.monotonic()
|
||||
compose('start', 'api')
|
||||
back = wait_in_rotation(container)
|
||||
result = {
|
||||
'replica': instance, 'mode': mode, 'action_seconds': round(action_seconds, 2),
|
||||
'target_served_until_s': round(max(r[1] for r in target) - acted, 2) if target else 0.0,
|
||||
'first_success_on_other_s': round(min(r[1] for r in others) - acted, 2) if others else None,
|
||||
'before': load.window(acted - WARMUP, acted),
|
||||
'during_10s': load.window(acted, acted + AFTER),
|
||||
'back_in_rotation_s': round(back - restored, 2),
|
||||
'data': load.verify(alice['token']),
|
||||
}
|
||||
print('%-12s %-4s errors=%-3d rps %5.1f -> %5.1f target last answer +%.2fs restore %.1fs lost=%d dup=%d'
|
||||
% (instance, mode, result['during_10s']['errors'], result['before']['rps'], result['during_10s']['rps'],
|
||||
result['target_served_until_s'], result['back_in_rotation_s'], result['data']['lost'],
|
||||
result['data']['duplicate_texts']))
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output')
|
||||
parser.add_argument('--modes', default='stop,kill')
|
||||
args = parser.parse_args()
|
||||
replicas = api_containers()
|
||||
assert len(replicas) >= 2, 'need at least two running API replicas'
|
||||
results = [run(container, mode) for mode in args.modes.split(',') for container in replicas]
|
||||
bad = [r for r in results if r['data']['lost'] or r['data']['duplicate_texts'] or r['data']['duplicate_ids']]
|
||||
summary = {'replicas': [c[:12] for c in replicas], 'warmup_s': WARMUP, 'observe_s': AFTER, 'workers': 4,
|
||||
'lb_check': 'GET /health/ready every 1s, fall 2, rise 2', 'results': results}
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as file:
|
||||
json.dump(summary, file, ensure_ascii=False, indent=2)
|
||||
print('FAIL: data loss or duplicates' if bad else 'PASS: every replica stopped once per mode; no loss, no duplicates')
|
||||
raise SystemExit(1 if bad else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Usage (стенд уже запущен через docker compose up -d):
|
||||
python student-tests/failure_check.py
|
||||
|
||||
Останавливает и замораживает Valkey и PostgreSQL (docker compose stop/pause, без удаления volumes),
|
||||
проверяет поведение API и восстановление.
|
||||
"""
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from helpers import basic, call, compose, new_chat, new_user, wait_ready
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def timed(method, path, token=None, body=None, headers=None):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status = call(method, path, body=body, token=token, headers=headers, timeout=10)[0]
|
||||
except OSError as error:
|
||||
status = type(error).__name__
|
||||
return status, time.monotonic() - started
|
||||
|
||||
|
||||
def expect(title, result, allowed, max_seconds):
|
||||
status, seconds = result
|
||||
ok = status in allowed and seconds <= max_seconds
|
||||
print(' %-48s -> %-4s за %.2f с %s' % (title, status, seconds, 'OK' if ok else 'FAIL'))
|
||||
if not ok:
|
||||
failures.append(title)
|
||||
|
||||
|
||||
def main():
|
||||
alice = new_user('failure')
|
||||
chat = new_chat(alice)
|
||||
history = '/api/v1/chats/%s/messages' % chat['id']
|
||||
assert call('POST', history, {'text': 'до отказа'}, alice['token'])[0] == 201
|
||||
fake = uuid.uuid4().hex * 2
|
||||
|
||||
print('1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL')
|
||||
compose('stop', 'valkey')
|
||||
try:
|
||||
expect('первый запрос: обнаружение отказа кэша', timed('GET', '/api/v1/users/me', alice['token']), [200], 1.5)
|
||||
expect('GET /users/me действующим токеном', timed('GET', '/api/v1/users/me', alice['token']), [200], 1)
|
||||
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'без кэша'}), [201], 1)
|
||||
expect('неизвестный токен', timed('GET', '/api/v1/users/me', fake), [401], 1)
|
||||
expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1)
|
||||
expect('/health/ready (Valkey некритичен)', timed('GET', '/health/ready'), [200], 1)
|
||||
started = time.monotonic()
|
||||
other = new_user('nocache')
|
||||
expect('регистрация и вход без кэша', (200, time.monotonic() - started), [200], 1)
|
||||
expect('logout без кэша', timed('DELETE', '/api/v1/auth/sessions/current', other['token']), [204], 1.5)
|
||||
expect('отозванный токен', timed('GET', '/api/v1/users/me', other['token']), [401], 1)
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 'valkey')
|
||||
|
||||
print('1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша')
|
||||
revoked = new_user('paused')
|
||||
call('GET', '/api/v1/users/me', token=revoked['token'])
|
||||
compose('pause', 'valkey')
|
||||
try:
|
||||
expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', revoked['token']), [204], 1.5)
|
||||
finally:
|
||||
compose('unpause', 'valkey')
|
||||
time.sleep(6)
|
||||
expect('через 6 с после unpause: /users/me', timed('GET', '/api/v1/users/me', revoked['token']), [401], 1)
|
||||
expect('через 6 с после unpause: сообщение', timed('POST', history, revoked['token'], {'text': 'после logout'}), [401], 1)
|
||||
|
||||
print('1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса')
|
||||
restarted = new_user('restart')
|
||||
call('GET', '/api/v1/users/me', token=restarted['token'])
|
||||
compose('pause', 'valkey')
|
||||
try:
|
||||
expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', restarted['token']), [204], 1.5)
|
||||
compose('restart', 'api')
|
||||
wait_ready()
|
||||
finally:
|
||||
compose('unpause', 'valkey')
|
||||
time.sleep(6)
|
||||
expect('после рестарта API и unpause: /users/me', timed('GET', '/api/v1/users/me', restarted['token']), [401], 1)
|
||||
expect('после рестарта API и unpause: сообщение', timed('POST', history, restarted['token'], {'text': 'после рестарта'}), [401], 1)
|
||||
|
||||
print('2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит')
|
||||
compose('stop', 'postgres')
|
||||
stopped_at = time.monotonic()
|
||||
try:
|
||||
expect('GET история', timed('GET', history, alice['token']), [503], 3)
|
||||
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при отказе БД'}), [503], 3)
|
||||
expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 3)
|
||||
expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1)
|
||||
expect('вход по Basic верным паролем', timed('POST', '/api/v1/auth/sessions',
|
||||
headers=basic(alice['username'], alice['password'])), [503], 3)
|
||||
expect('/health/live', timed('GET', '/health/live'), [200], 1)
|
||||
expect('/health/ready', timed('GET', '/health/ready'), [503], 3)
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 'postgres')
|
||||
ready_at = wait_ready()
|
||||
print('3. PostgreSQL запущен: готовность через %.1f с после остановки' % (ready_at - stopped_at))
|
||||
status = call('GET', '/api/v1/users/me', token=alice['token'])[0]
|
||||
texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']]
|
||||
print(' прежний токен -> %d; история: %s' % (status, texts))
|
||||
if status != 200 or texts != ['до отказа', 'без кэша']:
|
||||
failures.append('восстановление')
|
||||
|
||||
print('4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают')
|
||||
compose('pause', 'postgres')
|
||||
try:
|
||||
expect('GET история', timed('GET', history, alice['token']), [503], 4.5)
|
||||
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при заморозке'}), [503], 4.5)
|
||||
expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 4.5)
|
||||
expect('вход по Basic', timed('POST', '/api/v1/auth/sessions', headers=basic(alice['username'], alice['password'])), [503], 4.5)
|
||||
expect('/health/ready', timed('GET', '/health/ready'), [503], 2)
|
||||
expect('/health/live', timed('GET', '/health/live'), [200], 1)
|
||||
finally:
|
||||
compose('unpause', 'postgres')
|
||||
unpaused = time.monotonic()
|
||||
wait_ready()
|
||||
texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']]
|
||||
print(' готовность через %.1f с после unpause; история: %s' % (time.monotonic() - unpaused, texts))
|
||||
if texts != ['до отказа', 'без кэша']:
|
||||
failures.append('после заморозки')
|
||||
|
||||
print('FAIL: ' + ', '.join(failures) if failures else 'PASS: отказы Valkey и PostgreSQL обработаны')
|
||||
raise SystemExit(1 if failures else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
@@ -0,0 +1,191 @@
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, raw=None, headers=None, timeout=10):
|
||||
all_headers = {}
|
||||
if token:
|
||||
all_headers['Authorization'] = 'Bearer ' + token
|
||||
if body is not None:
|
||||
raw = json.dumps(body, ensure_ascii=False).encode('utf-8')
|
||||
if raw is not None:
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
all_headers.update(headers or {})
|
||||
if method == 'POST' and path.split('?')[0].endswith('/messages') and 'Idempotency-Key' not in all_headers:
|
||||
all_headers['Idempotency-Key'] = uuid.uuid4().hex
|
||||
all_headers = {name: value for name, value in all_headers.items() if value is not None}
|
||||
request = urllib.request.Request(BASE_URL + path, data=raw, headers=all_headers, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
data = response.read()
|
||||
is_json = response.headers.get_content_type() == 'application/json'
|
||||
return response.status, response.headers, json.loads(data) if data and is_json else None
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode('utf-8')).decode()}
|
||||
|
||||
|
||||
def register(label='user', password=None):
|
||||
username = (label + '_' + uuid.uuid4().hex[:12])[:32]
|
||||
password = password or 'Pw-' + uuid.uuid4().hex
|
||||
status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
|
||||
assert status == 201, status
|
||||
return user, password
|
||||
|
||||
|
||||
def login(username, password):
|
||||
status, _, session = call('POST', '/api/v1/auth/sessions', headers=basic(username, password))
|
||||
assert status == 201, status
|
||||
return session
|
||||
|
||||
|
||||
def new_user(label='user'):
|
||||
user, password = register(label)
|
||||
session = login(user['username'], password)
|
||||
return dict(user, password=password, token=session['token'])
|
||||
|
||||
|
||||
def new_chat(owner, *others, title='Тестовый чат'):
|
||||
members = [owner['id']] + [u['id'] for u in others]
|
||||
status, _, chat = call('POST', '/api/v1/chats', {'title': title, 'member_ids': members}, owner['token'])
|
||||
assert status == 201, status
|
||||
return chat
|
||||
|
||||
|
||||
def read_all_messages(chat_id, token, limit):
|
||||
items, cursor, pages = [], None, 0
|
||||
while True:
|
||||
query = '?limit=%d' % limit + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '')
|
||||
status, _, page = call('GET', '/api/v1/chats/%s/messages%s' % (chat_id, query), token=token)
|
||||
assert status == 200, status
|
||||
items += page['items']
|
||||
pages += 1
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return items, pages
|
||||
|
||||
|
||||
def compose(*args, env=None, check=True):
|
||||
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', env=env)
|
||||
if check and result.returncode != 0:
|
||||
raise RuntimeError('docker compose %s failed: %s' % (' '.join(args), result.stderr.strip()))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def sql(query, db='messenger', service='postgres'):
|
||||
return compose('exec', '-T', service, 'psql', '-U', 'messenger', '-d', db, '-X', '-tA', '-v', 'ON_ERROR_STOP=1',
|
||||
'-c', query)
|
||||
|
||||
|
||||
def wait_ready(seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if call('GET', '/health/ready', timeout=2)[0] == 200:
|
||||
return time.monotonic()
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
raise AssertionError('API is not ready after %ds' % seconds)
|
||||
|
||||
|
||||
def api_containers():
|
||||
return compose('ps', '-q', 'api').split()
|
||||
|
||||
|
||||
def wait_in_rotation(container, seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if call('GET', '/health/ready', timeout=2)[1].get('X-Instance-Id') == container[:12]:
|
||||
return time.monotonic()
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise AssertionError('container %s did not enter LB rotation' % container[:12])
|
||||
|
||||
|
||||
class Load:
|
||||
"""Отправляет сообщения в несколько потоков; неудачный POST повторяет с тем же Idempotency-Key."""
|
||||
|
||||
def __init__(self, token, chat_id, workers=4):
|
||||
self.token, self.path, self.workers = token, '/api/v1/chats/%s/messages' % chat_id, workers
|
||||
self.records, self.confirmed, self.uncertain = [], [], 0
|
||||
self.stopped = threading.Event()
|
||||
self.threads = []
|
||||
|
||||
def start(self):
|
||||
self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)]
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.stopped.set()
|
||||
for thread in self.threads:
|
||||
thread.join()
|
||||
|
||||
def worker(self, n):
|
||||
sent = 0
|
||||
while not self.stopped.is_set():
|
||||
key, text = uuid.uuid4().hex, 'load %d-%d %s' % (n, sent, uuid.uuid4().hex[:6])
|
||||
sent += 1
|
||||
while True:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status, headers, body = call('POST', self.path, {'text': text}, self.token,
|
||||
headers={'Idempotency-Key': key}, timeout=5)
|
||||
instance = headers.get('X-Instance-Id')
|
||||
except OSError as error:
|
||||
status, instance, body = type(error).__name__, None, None
|
||||
self.records.append((started, time.monotonic(), status, instance))
|
||||
if status == 201:
|
||||
self.confirmed.append(body)
|
||||
break
|
||||
if self.stopped.is_set():
|
||||
self.uncertain += 1
|
||||
break
|
||||
time.sleep(0.05)
|
||||
|
||||
def window(self, start, end):
|
||||
rows = [r for r in self.records if start <= r[0] < end]
|
||||
ok = [r for r in rows if r[2] == 201]
|
||||
latencies = sorted((r[1] - r[0]) * 1000 for r in ok)
|
||||
p = lambda q: round(latencies[min(len(latencies) - 1, int(len(latencies) * q))], 1) if latencies else None
|
||||
return {'seconds': round(end - start, 2), 'ok': len(ok), 'errors': len(rows) - len(ok),
|
||||
'rps': round(len(ok) / (end - start), 1) if end > start else 0,
|
||||
'p50_ms': p(0.5), 'p95_ms': p(0.95), 'p99_ms': p(0.99),
|
||||
'error_kinds': sorted({str(r[2]) for r in rows if r[2] != 201})}
|
||||
|
||||
def verify(self, token):
|
||||
stored, cursor = [], None
|
||||
while True:
|
||||
query = '?limit=100' + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '')
|
||||
page = call('GET', self.path + query, token=token)[2]
|
||||
stored += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
break
|
||||
ids = [m['id'] for m in stored]
|
||||
texts = [m['text'] for m in stored]
|
||||
return {'confirmed': len(self.confirmed), 'stored': len(stored),
|
||||
'lost': len({m['id'] for m in self.confirmed} - set(ids)),
|
||||
'duplicate_ids': len(ids) - len(set(ids)), 'duplicate_texts': len(texts) - len(set(texts)),
|
||||
'uncertain_at_stop': self.uncertain}
|
||||
@@ -0,0 +1,66 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE_PNG = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||
SAMPLE_JPG = (Path(__file__).resolve().parent / 'fixtures' / 'sample.jpg').read_bytes()
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
||||
|
||||
|
||||
def png(width, height, rows, color_type=0, level=9):
|
||||
"""rows — итератор байтовых строк пикселей (без байта фильтра)."""
|
||||
compressor = zlib.compressobj(level)
|
||||
parts = []
|
||||
for row in rows:
|
||||
parts.append(compressor.compress(b'\x00' + row))
|
||||
parts.append(compressor.flush())
|
||||
header = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def bomb(width=10000, height=5000):
|
||||
"""50 Мп чёрного: в распакованном виде 50 МБ, в файле — несколько десятков КБ."""
|
||||
row = bytes(width)
|
||||
return png(width, height, (row for _ in range(height)))
|
||||
|
||||
|
||||
def noise_png(target_bytes):
|
||||
"""Шумный RGB PNG (почти не сжимается) размером чуть меньше target_bytes."""
|
||||
width = 1024
|
||||
height = max(1, (target_bytes - 4096) // (width * 3 + 1))
|
||||
return png(width, height, (os.urandom(width * 3) for _ in range(height)), color_type=2, level=0)
|
||||
|
||||
|
||||
def broken_crc(data):
|
||||
"""Испортить один байт внутри IDAT, не пересчитывая CRC."""
|
||||
position = data.index(b'IDAT') + 100
|
||||
return data[:position] + bytes([data[position] ^ 0xff]) + data[position + 1:]
|
||||
|
||||
|
||||
def with_idat(data, transform):
|
||||
"""Заменить пиксельные данные PNG (все IDAT) на transform(idat), с правильными CRC."""
|
||||
position, chunks = 8, []
|
||||
while position < len(data):
|
||||
length, kind = struct.unpack('>I4s', data[position:position + 8])
|
||||
chunks.append((kind, data[position + 8:position + 8 + length]))
|
||||
position += 12 + length
|
||||
idat = b''.join(body for kind, body in chunks if kind == b'IDAT')
|
||||
first = next(index for index, (kind, _) in enumerate(chunks) if kind == b'IDAT')
|
||||
kept = [(kind, body) for kind, body in chunks if kind != b'IDAT']
|
||||
kept.insert(first, (b'IDAT', transform(idat)))
|
||||
return data[:8] + b''.join(chunk(kind, body) for kind, body in kept)
|
||||
|
||||
|
||||
def garbage_idat(data):
|
||||
"""Мусор вместо сжатых пикселей, CRC верные."""
|
||||
return with_idat(data, lambda idat: idat[:2] + os.urandom(len(idat) - 2))
|
||||
|
||||
|
||||
def short_idat(data):
|
||||
"""Корректный zlib-поток, но пикселей меньше, чем требует IHDR."""
|
||||
return with_idat(data, lambda idat: zlib.compress(zlib.decompress(idat)[:-500]))
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Usage (стенд запущен, обе реплики в ротации):
|
||||
python student-tests/rolling_update_check.py --output evidence/rolling-update.json
|
||||
|
||||
Под непрерывной нагрузкой (4 потока, повтор с тем же Idempotency-Key) выполняет scripts/rolling_update.py
|
||||
и проверяет: ошибки клиента, RPS, что все реплики заменены, что подтверждённые сообщения не потеряны и не задвоены.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from helpers import Load, api_containers, new_chat, new_user
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output')
|
||||
parser.add_argument('--no-build', action='store_true')
|
||||
args = parser.parse_args()
|
||||
before = api_containers()
|
||||
alice = new_user('rolling')
|
||||
chat = new_chat(alice)
|
||||
load = Load(alice['token'], chat['id']).start()
|
||||
time.sleep(5)
|
||||
started = time.monotonic()
|
||||
command = [sys.executable, str(ROOT / 'scripts' / 'rolling_update.py')] + (['--no-build'] if args.no_build else [])
|
||||
update = subprocess.run(command, capture_output=True, text=True, encoding='utf-8')
|
||||
finished = time.monotonic()
|
||||
time.sleep(5)
|
||||
load.stop()
|
||||
print(update.stdout.strip())
|
||||
after = api_containers()
|
||||
result = {
|
||||
'replicas_before': [c[:12] for c in before], 'replicas_after': [c[:12] for c in after],
|
||||
'update_seconds': round(finished - started, 1), 'update_exit_code': update.returncode,
|
||||
'before_5s': load.window(started - 5, started),
|
||||
'during_update': load.window(started, finished),
|
||||
'after_5s': load.window(finished, finished + 5),
|
||||
'data': load.verify(alice['token']),
|
||||
'log': update.stdout.strip().splitlines(),
|
||||
}
|
||||
replaced = not set(before) & set(after) and len(after) == len(before)
|
||||
ok = update.returncode == 0 and replaced and not result['data']['lost'] and not result['data']['duplicate_texts']
|
||||
print('rolling update %.1fs: errors=%d, rps %.1f -> %.1f -> %.1f, lost=%d, duplicates=%d, all replaced=%s'
|
||||
% (result['update_seconds'], result['during_update']['errors'], result['before_5s']['rps'],
|
||||
result['during_update']['rps'], result['after_5s']['rps'], result['data']['lost'],
|
||||
result['data']['duplicate_texts'], replaced))
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as file:
|
||||
json.dump(result, file, ensure_ascii=False, indent=2)
|
||||
print('PASS' if ok else 'FAIL: ' + update.stderr.strip()[-500:])
|
||||
raise SystemExit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Usage (стенд запущен через docker compose up -d):
|
||||
python student-tests/sigterm_check.py
|
||||
docker compose start api
|
||||
|
||||
Останавливает одну API-реплику (docker stop = SIGTERM) и всё это время опрашивает API через LB:
|
||||
реплика сначала выходит из ротации (readiness 503), потом завершается; клиенты не должны получить ошибок.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
from helpers import BASE_URL, api_containers, call, new_chat, new_user
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.run(['docker', *args], capture_output=True, text=True, encoding='utf-8').stdout.strip()
|
||||
|
||||
|
||||
def probe_forever(results, stop_event):
|
||||
while not stop_event.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + '/health/live', timeout=1) as response:
|
||||
outcome = str(response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
outcome = str(error.code)
|
||||
except Exception as error:
|
||||
outcome = type(error).__name__
|
||||
results.append((started, outcome))
|
||||
time.sleep(0.02)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--container', default=None)
|
||||
parser.add_argument('--stop-timeout', type=int, default=10, help='секунд до SIGKILL у docker stop')
|
||||
args = parser.parse_args()
|
||||
args.container = args.container or api_containers()[0]
|
||||
|
||||
marker = 'PRIVATE-' + uuid.uuid4().hex
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
status = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': marker}, alice['token'])[0]
|
||||
logs_before = docker('logs', args.container)
|
||||
print('сообщение с маркером отправлено: HTTP %d; маркер в логах: %s' % (status, marker in logs_before))
|
||||
|
||||
results, stop_event = [], threading.Event()
|
||||
prober = threading.Thread(target=probe_forever, args=(results, stop_event))
|
||||
prober.start()
|
||||
time.sleep(1)
|
||||
|
||||
sigterm_at = time.monotonic()
|
||||
docker('stop', '-t', str(args.stop_timeout), args.container)
|
||||
stopped_at = time.monotonic()
|
||||
time.sleep(1)
|
||||
stop_event.set()
|
||||
prober.join()
|
||||
|
||||
before = [r for t, r in results if t < sigterm_at]
|
||||
after = [r for t, r in results if t >= sigterm_at]
|
||||
exit_code = docker('inspect', '--format', '{{.State.ExitCode}}', args.container)
|
||||
print('docker stop занял: %.2f с (лимит %d с)' % (stopped_at - sigterm_at, args.stop_timeout))
|
||||
print('код выхода контейнера: %s (0 — сам завершился по SIGTERM; 137 — убит SIGKILL)' % exit_code)
|
||||
print('запросы до SIGTERM: %d, все 200: %s' % (len(before), set(before) == {'200'}))
|
||||
print('запросы после SIGTERM: %d, результаты: %s' % (len(after), sorted(set(after))))
|
||||
print('ответов 5xx за всё время: %d' % sum(r.startswith('5') for _, r in results))
|
||||
|
||||
print('\nлоги остановки:')
|
||||
for line in docker('logs', args.container).splitlines():
|
||||
if any(word in line for word in ('draining', 'Shutting down', 'stopping', 'shutdown', 'Finished server')):
|
||||
print(' ' + line)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,157 @@
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, login, new_chat, new_user, read_all_messages
|
||||
|
||||
|
||||
class ErrorFormatMixin:
|
||||
def assertError(self, result, status, code):
|
||||
actual, headers, body = result
|
||||
self.assertEqual(actual, status)
|
||||
self.assertEqual(body['error']['code'], code)
|
||||
self.assertEqual(body['error']['request_id'], headers['X-Request-Id'])
|
||||
self.assertNotIn('Traceback', body['error']['message'])
|
||||
|
||||
|
||||
class InvalidJsonTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def test_malformed_json(self):
|
||||
"""Битый JSON -> 400 invalid_request"""
|
||||
self.assertError(call('POST', '/api/v1/users', raw=b'{"username": "abc",'), 400, 'invalid_request')
|
||||
|
||||
def test_json_not_object(self):
|
||||
"""JSON-массив вместо объекта -> 400"""
|
||||
self.assertError(call('POST', '/api/v1/users', raw=b'[1, 2, 3]'), 400, 'invalid_request')
|
||||
|
||||
def test_wrong_type(self):
|
||||
"""Число вместо строки -> 400 (без тихого приведения типов)"""
|
||||
self.assertError(call('POST', '/api/v1/users', {'username': 12345, 'display_name': 'x', 'password': 'Long-enough-pass-1'}), 400, 'invalid_request')
|
||||
|
||||
def test_extra_field(self):
|
||||
"""Лишнее поле в теле -> 400"""
|
||||
body = {'username': 'u_' + uuid.uuid4().hex[:8], 'display_name': 'x', 'password': 'Long-enough-pass-1', 'is_admin': True}
|
||||
self.assertError(call('POST', '/api/v1/users', body), 400, 'invalid_request')
|
||||
|
||||
|
||||
class DuplicateUsernameTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def test_duplicate_username(self):
|
||||
"""Повторная регистрация того же username -> 409 conflict"""
|
||||
user = new_user('dup')
|
||||
again = call('POST', '/api/v1/users', {'username': user['username'], 'display_name': 'Другой',
|
||||
'password': 'Other-password-123'})
|
||||
self.assertError(again, 409, 'conflict')
|
||||
|
||||
|
||||
class ForeignChatTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
self.path = '/api/v1/chats/' + self.chat['id']
|
||||
|
||||
def test_outsider_cannot_read_chat(self):
|
||||
"""Посторонний читает чат -> 404, как будто чата нет"""
|
||||
self.assertError(call('GET', self.path, token=self.eve['token']), 404, 'not_found')
|
||||
missing = call('GET', '/api/v1/chats/' + str(uuid.uuid4()), token=self.eve['token'])
|
||||
self.assertEqual(missing[2]['error']['message'], call('GET', self.path, token=self.eve['token'])[2]['error']['message'])
|
||||
|
||||
def test_outsider_cannot_write_or_read_history(self):
|
||||
"""Посторонний пишет/читает историю -> 404"""
|
||||
self.assertError(call('POST', self.path + '/messages', {'text': 'чужое'}, self.eve['token']), 404, 'not_found')
|
||||
self.assertError(call('GET', self.path + '/messages', token=self.eve['token']), 404, 'not_found')
|
||||
|
||||
def test_outsider_does_not_see_chat_in_list(self):
|
||||
"""Чужой чат не попадает в список чатов"""
|
||||
status, _, page = call('GET', '/api/v1/chats?limit=100', token=self.eve['token'])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertNotIn(self.chat['id'], [c['id'] for c in page['items']])
|
||||
|
||||
def test_member_has_access(self):
|
||||
"""Участник видит чат (контроль: 404 выше не из-за ошибки в тесте)"""
|
||||
self.assertEqual(call('GET', self.path, token=self.bob['token'])[0], 200)
|
||||
|
||||
|
||||
class EmptyMessageTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.path = '/api/v1/chats/%s/messages' % new_chat(self.alice)['id']
|
||||
|
||||
def test_empty_text(self):
|
||||
"""Пустой текст -> 400"""
|
||||
self.assertError(call('POST', self.path, {'text': ''}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_whitespace_text(self):
|
||||
"""Текст только из пробельных символов -> 400"""
|
||||
self.assertError(call('POST', self.path, {'text': ' \t\n '}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_missing_text(self):
|
||||
"""Тело без поля text -> 400"""
|
||||
self.assertError(call('POST', self.path, {}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_text_length_limit(self):
|
||||
"""4000 символов можно (эмодзи = 1 символ), 4001 — 400"""
|
||||
self.assertEqual(call('POST', self.path, {'text': '😀' * 4000}, self.alice['token'])[0], 201)
|
||||
self.assertError(call('POST', self.path, {'text': 'x' * 4001}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
|
||||
class PaginationTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.chat = new_chat(self.alice)
|
||||
self.sent = [call('POST', '/api/v1/chats/%s/messages' % self.chat['id'],
|
||||
{'text': 'сообщение %d' % i}, self.alice['token'])[2] for i in range(7)]
|
||||
|
||||
def test_walk_several_pages(self):
|
||||
"""7 сообщений по 3: три страницы, без потерь, повторов и перестановок"""
|
||||
items, pages = read_all_messages(self.chat['id'], self.alice['token'], limit=3)
|
||||
self.assertEqual(pages, 3)
|
||||
self.assertEqual([m['id'] for m in items], [m['id'] for m in self.sent])
|
||||
keys = [(m['created_at'], m['id']) for m in items]
|
||||
self.assertEqual(keys, sorted(keys))
|
||||
|
||||
def test_cursor_from_other_chat(self):
|
||||
"""cursor от другого чата -> 400"""
|
||||
_, _, page = call('GET', '/api/v1/chats/%s/messages?limit=1' % self.chat['id'], token=self.alice['token'])
|
||||
other = new_chat(self.alice, title='Другой')
|
||||
self.assertError(call('GET', '/api/v1/chats/%s/messages?cursor=%s' % (other['id'], page['next_cursor']),
|
||||
token=self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_invalid_cursor_and_limit(self):
|
||||
"""Мусорный cursor и limit вне 1..100 -> 400"""
|
||||
path = '/api/v1/chats/%s/messages' % self.chat['id']
|
||||
for query in ('?cursor=not-a-cursor', '?limit=0', '?limit=101', '?limit=abc'):
|
||||
with self.subTest(query=query):
|
||||
self.assertError(call('GET', path + query, token=self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
|
||||
class IdentityTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def test_sender_cannot_be_forged(self):
|
||||
"""sender_id в теле запрещён: отправитель только из сессии"""
|
||||
alice, bob = new_user('alice'), new_user('bob')
|
||||
chat = new_chat(alice, bob)
|
||||
forged = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': 'hi', 'sender_id': bob['id']}, alice['token'])
|
||||
self.assertError(forged, 400, 'invalid_request')
|
||||
|
||||
def test_nul_character_is_preserved(self):
|
||||
"""U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений"""
|
||||
username, password = 'nul_' + uuid.uuid4().hex[:12], 'Nul-password-123'
|
||||
status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'a\u0000lice', 'password': password})
|
||||
self.assertEqual((status, user['display_name']), (201, 'a\u0000lice'))
|
||||
alice = dict(user, token=login(username, password)['token'])
|
||||
self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[2]['display_name'], 'a\u0000lice')
|
||||
status, _, chat = call('POST', '/api/v1/chats', {'title': 't\u0000', 'member_ids': [alice['id']]}, alice['token'])
|
||||
self.assertEqual((status, chat['title']), (201, 't\u0000'))
|
||||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||||
status, _, message = call('POST', path, {'text': 'a\u0000b'}, alice['token'])
|
||||
self.assertEqual((status, message['text']), (201, 'a\u0000b'))
|
||||
self.assertEqual(call('GET', path, token=alice['token'])[2]['items'][0]['text'], 'a\u0000b')
|
||||
self.assertEqual(call('GET', '/api/v1/chats/' + chat['id'], token=alice['token'])[2]['title'], 't\u0000')
|
||||
|
||||
def test_lone_surrogate_is_rejected(self):
|
||||
"""Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500"""
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
self.assertError(call('POST', '/api/v1/chats/%s/messages' % chat['id'], raw=b'{"text": "a\\ud800b"}',
|
||||
token=alice['token']), 400, 'invalid_request')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,192 @@
|
||||
import hashlib
|
||||
import http.client
|
||||
import unittest
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
from helpers import BASE_URL, call, new_chat, new_user
|
||||
from imagegen import SAMPLE_JPG, SAMPLE_PNG, bomb, broken_crc, garbage_idat, noise_png, short_idat
|
||||
|
||||
MAX = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def upload(user, chat, data, content_type='image/png', filename='photo.png', token=True):
|
||||
query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='')
|
||||
return call('POST', '/api/v1/chats/%s/attachments%s' % (chat['id'], query), raw=data,
|
||||
token=user['token'] if token else None, headers={'Content-Type': content_type})
|
||||
|
||||
|
||||
def download(user, attachment_id):
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=30)
|
||||
connection.request('GET', '/api/v1/attachments/%s/content' % attachment_id,
|
||||
headers={'Authorization': 'Bearer ' + user['token']})
|
||||
response = connection.getresponse()
|
||||
body = response.read()
|
||||
connection.close()
|
||||
return response.status, response.headers, body
|
||||
|
||||
|
||||
def send(user, chat, body, key=None):
|
||||
return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'],
|
||||
headers={'Idempotency-Key': key or uuid.uuid4().hex})
|
||||
|
||||
|
||||
class UploadTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
|
||||
def assertCode(self, result, status, code=None):
|
||||
self.assertEqual(result[0], status, result[2])
|
||||
if code:
|
||||
self.assertEqual(result[2]['error']['code'], code)
|
||||
|
||||
def test_png_and_jpeg_are_stored_and_downloaded_byte_exact(self):
|
||||
"""PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type"""
|
||||
for data, content_type in ((SAMPLE_PNG, 'image/png'), (SAMPLE_JPG, 'image/jpeg')):
|
||||
with self.subTest(content_type=content_type):
|
||||
status, _, att = upload(self.alice, self.chat, data, content_type)
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual((att['status'], att['variants'], att['error_code']), ('ready', [], None))
|
||||
self.assertEqual((att['content_type'], att['size_bytes'], att['sha256']),
|
||||
(content_type, len(data), hashlib.sha256(data).hexdigest()))
|
||||
self.assertEqual((att['chat_id'], att['owner_id']), (self.chat['id'], self.alice['id']))
|
||||
self.assertEqual(call('GET', '/api/v1/attachments/' + att['id'], token=self.bob['token'])[2], att)
|
||||
status, headers, body = download(self.bob, att['id'])
|
||||
self.assertEqual((status, headers.get_content_type(), body), (200, content_type, data))
|
||||
self.assertEqual(int(headers['Content-Length']), len(data))
|
||||
|
||||
def test_download_through_every_replica(self):
|
||||
"""Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей)"""
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
replicas = set()
|
||||
for _ in range(6):
|
||||
status, headers, body = download(self.bob, att['id'])
|
||||
self.assertEqual((status, body), (200, SAMPLE_PNG))
|
||||
replicas.add(headers['X-Instance-Id'])
|
||||
self.assertGreaterEqual(len(replicas), 2)
|
||||
|
||||
def test_filename_is_display_only_and_validated(self):
|
||||
"""Unicode-имя сохраняется как есть; нет имени, пустое, /, \\, NUL, управляющий символ, >255 → 400"""
|
||||
status, _, att = upload(self.alice, self.chat, SAMPLE_PNG, filename='фото отпуска 😀.png')
|
||||
self.assertEqual((status, att['filename']), (201, 'фото отпуска 😀.png'))
|
||||
for name in (None, '', ' ', '../etc/passwd', 'a\\b.png', 'a\x00b.png', 'bell\x07.png', 'x' * 256):
|
||||
with self.subTest(name=name):
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, filename=name), 400, 'invalid_request')
|
||||
|
||||
def test_wrong_declared_type(self):
|
||||
"""Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415"""
|
||||
for content_type in ('application/octet-stream', 'multipart/form-data; boundary=x', 'image/gif', 'text/plain'):
|
||||
with self.subTest(content_type=content_type):
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, content_type), 415, 'unsupported_media_type')
|
||||
|
||||
def test_mime_spoofing(self):
|
||||
"""Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415"""
|
||||
for data, content_type in ((SAMPLE_PNG, 'image/jpeg'), (SAMPLE_JPG, 'image/png'),
|
||||
(b'hello, not an image', 'image/png'), (b'GIF89a' + bytes(100), 'image/png'),
|
||||
(b'', 'image/png')):
|
||||
with self.subTest(content_type=content_type, head=data[:6]):
|
||||
self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type')
|
||||
|
||||
def test_corrupted_images(self):
|
||||
"""Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415"""
|
||||
for data, content_type in ((SAMPLE_PNG[:5000], 'image/png'), (broken_crc(SAMPLE_PNG), 'image/png'),
|
||||
(garbage_idat(SAMPLE_PNG), 'image/png'), (short_idat(SAMPLE_PNG), 'image/png'),
|
||||
(SAMPLE_JPG[:3000], 'image/jpeg')):
|
||||
with self.subTest(size=len(data), content_type=content_type):
|
||||
self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type')
|
||||
|
||||
def test_too_large_file(self):
|
||||
"""10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается"""
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG + bytes(MAX + 1 - len(SAMPLE_PNG))), 413, 'payload_too_large')
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60)
|
||||
chunks = (SAMPLE_PNG if i == 0 else bytes(1024 * 1024) for i in range(12))
|
||||
connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % self.chat['id'], body=chunks,
|
||||
headers={'Authorization': 'Bearer ' + self.alice['token'], 'Content-Type': 'image/png'},
|
||||
encode_chunked=True)
|
||||
self.assertEqual(connection.getresponse().status, 413)
|
||||
connection.close()
|
||||
big = noise_png(MAX - 50000)
|
||||
self.assertLessEqual(len(big), MAX)
|
||||
status, _, att = upload(self.alice, self.chat, big)
|
||||
self.assertEqual((status, att['size_bytes']), (201, len(big)))
|
||||
self.assertEqual(download(self.bob, att['id'])[2], big)
|
||||
|
||||
def test_decompression_bomb(self):
|
||||
"""PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки"""
|
||||
data = bomb()
|
||||
self.assertLess(len(data), 1024 * 1024)
|
||||
self.assertCode(upload(self.alice, self.chat, data), 413, 'payload_too_large')
|
||||
|
||||
def test_access_rules(self):
|
||||
"""Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404)"""
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, token=False), 401, 'unauthorized')
|
||||
self.assertCode(upload(self.eve, self.chat, SAMPLE_PNG), 404, 'not_found')
|
||||
self.assertCode(upload(self.alice, {'id': str(uuid.uuid4())}, SAMPLE_PNG), 404, 'not_found')
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
self.assertCode(call('GET', '/api/v1/attachments/' + att['id'], token=self.eve['token']), 404, 'not_found')
|
||||
self.assertEqual(download(self.eve, att['id'])[0], 404)
|
||||
self.assertCode(call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), token=self.bob['token']), 404)
|
||||
self.assertCode(call('GET', '/api/v1/attachments/not-a-uuid', token=self.bob['token']), 400)
|
||||
self.assertCode(call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'],
|
||||
token=self.bob['token']), 400)
|
||||
|
||||
|
||||
class MessageAttachmentsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob = new_user('alice'), new_user('bob')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
self.other_chat = new_chat(self.alice, title='Другой')
|
||||
self.att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
|
||||
def history(self):
|
||||
return call('GET', '/api/v1/chats/%s/messages?limit=100' % self.chat['id'], token=self.bob['token'])[2]['items']
|
||||
|
||||
def test_message_with_attachment_only(self):
|
||||
"""text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает []"""
|
||||
status, _, message = send(self.alice, self.chat, {'text': '', 'attachment_ids': [self.att['id']]})
|
||||
self.assertEqual((status, message['attachment_ids'], message['text']), (201, [self.att['id']], ''))
|
||||
plain = send(self.alice, self.chat, {'text': 'без вложений'})[2]
|
||||
self.assertEqual(plain['attachment_ids'], [])
|
||||
self.assertEqual([m['attachment_ids'] for m in self.history()], [[self.att['id']], []])
|
||||
|
||||
def test_other_member_can_reference_attachment(self):
|
||||
"""Боб прикрепляет вложение, загруженное Алисой в этот же чат"""
|
||||
status, _, message = send(self.bob, self.chat, {'text': 'смотри', 'attachment_ids': [self.att['id']]})
|
||||
self.assertEqual((status, message['attachment_ids']), (201, [self.att['id']]))
|
||||
|
||||
def test_foreign_or_unknown_attachment(self):
|
||||
"""Вложение другого чата или несуществующее → 404, сообщение не создаётся"""
|
||||
foreign = upload(self.alice, self.other_chat, SAMPLE_PNG)[2]
|
||||
for ids in ([foreign['id']], [str(uuid.uuid4())], [self.att['id'], foreign['id']]):
|
||||
with self.subTest(ids=ids):
|
||||
self.assertEqual(send(self.alice, self.chat, {'text': 'x', 'attachment_ids': ids})[0], 404)
|
||||
self.assertEqual(self.history(), [])
|
||||
|
||||
def test_invalid_attachment_lists(self):
|
||||
"""Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400"""
|
||||
for body in ({'text': 'x', 'attachment_ids': [self.att['id'], self.att['id']]},
|
||||
{'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]},
|
||||
{'text': 'x', 'attachment_ids': ['nope']},
|
||||
{'text': ''}, {'text': ' '}, {'text': '', 'attachment_ids': []}):
|
||||
with self.subTest(body=body):
|
||||
self.assertEqual(send(self.alice, self.chat, body)[0], 400)
|
||||
|
||||
def test_idempotency_includes_attachments(self):
|
||||
"""Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста"""
|
||||
key = uuid.uuid4().hex
|
||||
first = send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [self.att['id']]}, key)
|
||||
again = send(self.alice, self.chat, {'attachment_ids': [self.att['id']], 'text': 'фото'}, key)
|
||||
self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2]))
|
||||
other = upload(self.alice, self.chat, SAMPLE_JPG, 'image/jpeg', 'b.jpg')[2]
|
||||
self.assertEqual(send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [other['id']]}, key)[0], 409)
|
||||
key = uuid.uuid4().hex
|
||||
plain = send(self.alice, self.chat, {'text': 'текст'}, key)
|
||||
same = send(self.alice, self.chat, {'text': 'текст', 'attachment_ids': []}, key)
|
||||
self.assertEqual(plain[2]['id'], same[2]['id'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,132 @@
|
||||
from datetime import datetime, timezone
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import basic, call, login, new_user, register
|
||||
|
||||
|
||||
class RegistrationTest(unittest.TestCase):
|
||||
def body(self, password):
|
||||
return {'username': 'reg_' + uuid.uuid4().hex[:12], 'display_name': 'Регистрация', 'password': password}
|
||||
|
||||
def test_password_is_not_returned(self):
|
||||
"""Ответ регистрации и /users/me не содержат password/hash"""
|
||||
status, _, user = call('POST', '/api/v1/users', self.body('Secret-password-1'))
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(set(user), {'id', 'username', 'display_name', 'created_at'})
|
||||
|
||||
def test_password_length_limits(self):
|
||||
"""Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом"""
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 11))[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 129))[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 128))[0], 201)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('🔑' * 12))[0], 201)
|
||||
|
||||
def test_password_required_and_not_blank(self):
|
||||
"""Без password или из одних пробелов -> 400"""
|
||||
body = self.body('x')
|
||||
del body['password']
|
||||
self.assertEqual(call('POST', '/api/v1/users', body)[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body(' ' * 20))[0], 400)
|
||||
|
||||
|
||||
class LoginTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.user, self.password = register('login')
|
||||
|
||||
def assertBasicChallenge(self, result):
|
||||
status, headers, body = result
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(headers['WWW-Authenticate'], 'Basic realm="messenger", charset="UTF-8"')
|
||||
self.assertEqual(body['error']['code'], 'unauthorized')
|
||||
return body['error']['message']
|
||||
|
||||
def test_login_returns_opaque_session(self):
|
||||
"""Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID"""
|
||||
first = login(self.user['username'], self.password)
|
||||
second = login(self.user['username'], self.password)
|
||||
self.assertEqual(first['token_type'], 'Bearer')
|
||||
self.assertEqual(first['user'], self.user)
|
||||
self.assertGreaterEqual(len(first['token']), 43)
|
||||
self.assertNotEqual(first['token'], second['token'])
|
||||
self.assertNotIn(self.user['username'], first['token'])
|
||||
self.assertNotIn(self.user['id'], first['token'])
|
||||
expires = datetime.fromisoformat(first['expires_at'].replace('Z', '+00:00'))
|
||||
self.assertGreater(expires, datetime.now(timezone.utc))
|
||||
|
||||
def test_wrong_password_and_unknown_user_look_the_same(self):
|
||||
"""Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge"""
|
||||
wrong = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions',
|
||||
headers=basic(self.user['username'], 'Wrong-password-1')))
|
||||
unknown = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions',
|
||||
headers=basic('nobody_' + uuid.uuid4().hex[:8], 'Wrong-password-1')))
|
||||
self.assertEqual(wrong, unknown)
|
||||
|
||||
def test_malformed_basic(self):
|
||||
"""Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic"""
|
||||
for header in (None, 'Basic', 'Basic !!!not-base64!!!', 'Basic ' + 'bm9jb2xvbg==', 'Bearer abc'):
|
||||
with self.subTest(header=header):
|
||||
headers = {'Authorization': header} if header else {}
|
||||
self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', headers=headers))
|
||||
|
||||
def test_password_with_colon_and_unicode(self):
|
||||
"""Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':'"""
|
||||
user, password = register('colon', password='пароль:с:двоеточием🔑')
|
||||
self.assertEqual(login(user['username'], password)['user']['id'], user['id'])
|
||||
|
||||
def test_login_rejects_body(self):
|
||||
"""POST /auth/sessions принимает только заголовок, JSON body -> 400"""
|
||||
status = call('POST', '/api/v1/auth/sessions', {'username': self.user['username']},
|
||||
headers=basic(self.user['username'], self.password))[0]
|
||||
self.assertEqual(status, 400)
|
||||
|
||||
|
||||
class BearerTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.eve = new_user('eve')
|
||||
|
||||
def assertBearerChallenge(self, headers):
|
||||
status, response_headers, _ = call('GET', '/api/v1/users/me', headers=headers)
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(response_headers['WWW-Authenticate'], 'Bearer realm="messenger"')
|
||||
|
||||
def test_only_bearer_is_accepted(self):
|
||||
"""Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer"""
|
||||
self.assertBearerChallenge({})
|
||||
self.assertBearerChallenge(basic(self.alice['username'], self.alice['password']))
|
||||
self.assertBearerChallenge({'X-User-Id': self.alice['id']})
|
||||
self.assertBearerChallenge({'Authorization': 'Bearer ' + uuid.uuid4().hex})
|
||||
self.assertBearerChallenge({'Authorization': 'Bearer'})
|
||||
|
||||
def test_authentication_before_validation(self):
|
||||
"""Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400"""
|
||||
self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad')[0], 401)
|
||||
self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad', token=self.alice['token'])[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', raw=b'{bad')[0], 400)
|
||||
|
||||
def test_scheme_is_case_insensitive(self):
|
||||
"""Схема аутентификации нечувствительна к регистру (RFC 9110)"""
|
||||
status, _, me = call('GET', '/api/v1/users/me', headers={'Authorization': 'bearer ' + self.alice['token']})
|
||||
self.assertEqual((status, me['id']), (200, self.alice['id']))
|
||||
|
||||
def test_x_user_id_does_not_override_session(self):
|
||||
"""X-User-Id игнорируется: владелец определяется только токеном"""
|
||||
status, _, me = call('GET', '/api/v1/users/me', token=self.alice['token'], headers={'X-User-Id': self.eve['id']})
|
||||
self.assertEqual((status, me['id']), (200, self.alice['id']))
|
||||
|
||||
|
||||
class LogoutTest(unittest.TestCase):
|
||||
def test_logout_revokes_only_current_session(self):
|
||||
"""Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает"""
|
||||
alice = new_user('alice')
|
||||
other = login(alice['username'], alice['password'])['token']
|
||||
status, _, body = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])
|
||||
self.assertEqual((status, body), (204, None))
|
||||
self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[0], 401)
|
||||
self.assertEqual(call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0], 401)
|
||||
self.assertEqual(call('GET', '/api/v1/users/me', token=other)[0], 200)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,68 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import basic, call, new_chat, new_user, read_all_messages
|
||||
|
||||
CLIENTS = 10
|
||||
REGISTRATIONS = 20
|
||||
PER_CLIENT = 10
|
||||
|
||||
|
||||
class ConcurrencyTest(unittest.TestCase):
|
||||
def test_100_messages_from_10_clients(self):
|
||||
"""100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории"""
|
||||
users = [new_user('client%d' % i) for i in range(CLIENTS)]
|
||||
chat = new_chat(users[0], *users[1:], title='Нагрузка')
|
||||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||||
barrier = threading.Barrier(CLIENTS)
|
||||
|
||||
def client(index):
|
||||
barrier.wait()
|
||||
results = []
|
||||
for n in range(PER_CLIENT):
|
||||
status, _, message = call('POST', path, {'text': 'client %d msg %d' % (index, n)}, users[index]['token'])
|
||||
results.append((status, message))
|
||||
return results
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
|
||||
responses = [r for batch in pool.map(client, range(CLIENTS)) for r in batch]
|
||||
|
||||
self.assertEqual([s for s, _ in responses], [201] * CLIENTS * PER_CLIENT)
|
||||
sent_ids = {m['id'] for _, m in responses}
|
||||
items, pages = read_all_messages(chat['id'], users[0]['token'], limit=30)
|
||||
read_ids = [m['id'] for m in items]
|
||||
keys = [(m['created_at'], m['id']) for m in items]
|
||||
|
||||
self.assertEqual(len(sent_ids), 100, 'сервер выдал повторяющиеся id')
|
||||
self.assertEqual(len(read_ids), 100, 'в истории не ровно 100 сообщений')
|
||||
self.assertEqual(set(read_ids), sent_ids, 'история не совпадает с подтверждёнными сообщениями')
|
||||
self.assertEqual(len({m['text'] for m in items}), 100, 'потерялся или задвоился текст')
|
||||
self.assertEqual(keys, sorted(keys), 'история не упорядочена по (created_at, id)')
|
||||
print('\n отправлено 201: %d, уникальных id: %d, прочитано: %d за %d страниц(ы)'
|
||||
% (len(responses), len(sent_ids), len(read_ids), pages))
|
||||
|
||||
def test_parallel_registration_of_same_username(self):
|
||||
"""20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя"""
|
||||
username = 'race_' + uuid.uuid4().hex[:12]
|
||||
barrier = threading.Barrier(REGISTRATIONS)
|
||||
|
||||
def register(index):
|
||||
password = 'Race-password-%02d' % index
|
||||
barrier.wait()
|
||||
status = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Гонка', 'password': password})[0]
|
||||
return status, password
|
||||
|
||||
with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool:
|
||||
results = list(pool.map(register, range(REGISTRATIONS)))
|
||||
statuses = sorted(status for status, _ in results)
|
||||
self.assertEqual(statuses, [201] + [409] * (REGISTRATIONS - 1))
|
||||
winner = [password for status, password in results if status == 201][0]
|
||||
loser = [password for status, password in results if status == 409][0]
|
||||
self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, winner))[0], 201)
|
||||
self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, loser))[0], 401)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
import hashlib
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, compose, new_chat, new_user, sql
|
||||
|
||||
|
||||
class DatabaseTest(unittest.TestCase):
|
||||
def test_chat_creation_is_rolled_back(self):
|
||||
"""Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано"""
|
||||
alice = new_user('alice')
|
||||
chats_before = int(sql('SELECT count(*) FROM chats'))
|
||||
members_before = int(sql('SELECT count(*) FROM chat_members'))
|
||||
ghost = str(uuid.uuid4())
|
||||
since = sql("SELECT to_char(now() - interval '1 second', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')")
|
||||
status, _, body = call('POST', '/api/v1/chats', {'title': 'Откат', 'member_ids': [alice['id'], ghost]},
|
||||
alice['token'])
|
||||
self.assertEqual(status, 400, body)
|
||||
time.sleep(0.5)
|
||||
log = compose('logs', '--no-log-prefix', '--since', since, 'postgres')
|
||||
self.assertIn('violates foreign key constraint', log)
|
||||
self.assertIn(ghost, log)
|
||||
self.assertEqual(int(sql('SELECT count(*) FROM chats')), chats_before)
|
||||
self.assertEqual(int(sql('SELECT count(*) FROM chat_members')), members_before)
|
||||
|
||||
def test_secrets_are_not_stored_in_plaintext(self):
|
||||
"""В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен"""
|
||||
alice = new_user('alice')
|
||||
password_hash = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id'])
|
||||
self.assertTrue(password_hash.startswith('$argon2id$'), password_hash[:10])
|
||||
self.assertNotIn(alice['password'], password_hash)
|
||||
digest = hashlib.sha256(alice['token'].encode()).hexdigest()
|
||||
self.assertEqual(sql("SELECT count(*) FROM sessions WHERE token_hash = '\\x%s'" % digest), '1')
|
||||
dump = sql("SELECT string_agg(encode(token_hash, 'escape'), '') FROM sessions WHERE user_id = '%s'" % alice['id'])
|
||||
self.assertNotIn(alice['token'], dump)
|
||||
|
||||
def test_migrations_are_idempotent(self):
|
||||
"""Повторный запуск миграций ничего не меняет и завершается успешно"""
|
||||
before = sql('SELECT version, applied_at FROM schema_migrations ORDER BY version')
|
||||
users = sql('SELECT count(*) FROM users')
|
||||
output = compose('run', '--rm', '--no-deps', 'migrate')
|
||||
self.assertIn('skip 0001_init: already applied', output)
|
||||
self.assertEqual(sql('SELECT version, applied_at FROM schema_migrations ORDER BY version'), before)
|
||||
self.assertEqual(sql('SELECT count(*) FROM users'), users)
|
||||
|
||||
def test_history_query_uses_index(self):
|
||||
"""Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort"""
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
plan = sql("SET enable_seqscan = off; SET enable_bitmapscan = off; EXPLAIN SELECT id FROM messages WHERE chat_id = '%s' "
|
||||
"ORDER BY created_at, id LIMIT 51" % chat['id'])
|
||||
self.assertIn('messages_chat_history_idx', plan)
|
||||
self.assertNotIn('Sort', plan)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,96 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, new_chat, new_user, read_all_messages
|
||||
|
||||
PARALLEL = 20
|
||||
|
||||
|
||||
def post(chat, user, key, body=None, raw=None):
|
||||
return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], raw=raw,
|
||||
headers={'Idempotency-Key': key})
|
||||
|
||||
|
||||
class IdempotencyTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob = new_user('alice'), new_user('bob')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
|
||||
def history(self):
|
||||
return read_all_messages(self.chat['id'], self.alice['token'], limit=100)[0]
|
||||
|
||||
def test_replay_returns_same_message(self):
|
||||
"""Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет"""
|
||||
key = uuid.uuid4().hex
|
||||
first = post(self.chat, self.alice, key, {'text': 'один раз'})
|
||||
second = post(self.chat, self.alice, key, {'text': 'один раз'})
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[2], second[2])
|
||||
self.assertEqual([m['id'] for m in self.history()], [first[2]['id']])
|
||||
|
||||
def test_json_formatting_does_not_matter(self):
|
||||
"""Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message"""
|
||||
key = uuid.uuid4().hex
|
||||
first = post(self.chat, self.alice, key, raw='{"text":"формат"}'.encode())
|
||||
second = post(self.chat, self.alice, key, raw='{ "text" : "формат" }'.encode())
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[2]['id'], second[2]['id'])
|
||||
|
||||
def test_other_body_with_same_key_conflicts(self):
|
||||
"""Тот же ключ с другим телом -> 409, второе сообщение не создаётся"""
|
||||
key = uuid.uuid4().hex
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'первое'})[0], 201)
|
||||
status, _, body = post(self.chat, self.alice, key, {'text': 'другое'})
|
||||
self.assertEqual((status, body['error']['code']), (409, 'conflict'))
|
||||
self.assertEqual(len(self.history()), 1)
|
||||
|
||||
def test_missing_or_invalid_key(self):
|
||||
"""Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400"""
|
||||
for key in (None, '', 'bad key', 'bad/key', 'café', 'x' * 129):
|
||||
with self.subTest(key=key):
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'x'})[0], 400)
|
||||
self.assertEqual(post(self.chat, self.alice, 'A.b_c:d-1' + 'x' * 119, {'text': 'граница'})[0], 201)
|
||||
|
||||
def test_key_scope_is_user_and_chat(self):
|
||||
"""Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения"""
|
||||
key = uuid.uuid4().hex
|
||||
other_chat = new_chat(self.alice, title='Другой')
|
||||
a = post(self.chat, self.alice, key, {'text': 'от Алисы'})
|
||||
b = post(self.chat, self.bob, key, {'text': 'от Боба'})
|
||||
c = post(other_chat, self.alice, key, {'text': 'от Алисы'})
|
||||
self.assertEqual([a[0], b[0], c[0]], [201, 201, 201])
|
||||
self.assertEqual(len({a[2]['id'], b[2]['id'], c[2]['id']}), 3)
|
||||
|
||||
def test_replay_rechecks_session_and_access(self):
|
||||
"""Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404"""
|
||||
key = uuid.uuid4().hex
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 201)
|
||||
eve = new_user('eve')
|
||||
self.assertEqual(post(self.chat, eve, key, {'text': 'до выхода'})[0], 404)
|
||||
call('DELETE', '/api/v1/auth/sessions/current', token=self.alice['token'])
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 401)
|
||||
|
||||
def test_parallel_retries_through_replicas_create_one_message(self):
|
||||
"""20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id"""
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(PARALLEL)
|
||||
|
||||
def send(_):
|
||||
barrier.wait()
|
||||
status, headers, body = post(self.chat, self.alice, key, {'text': 'конкурентный повтор'})
|
||||
return status, headers['X-Instance-Id'], body
|
||||
|
||||
with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
|
||||
results = list(pool.map(send, range(PARALLEL)))
|
||||
self.assertEqual({status for status, _, _ in results}, {201})
|
||||
self.assertEqual(len({body['id'] for _, _, body in results}), 1)
|
||||
self.assertGreaterEqual(len({instance for _, instance, _ in results}), 2, 'запросы не попали на разные реплики')
|
||||
self.assertEqual(len(self.history()), 1)
|
||||
print('\n 20 повторов: реплики %s, сообщений в истории: %d'
|
||||
% (sorted({instance for _, instance, _ in results}), len(self.history())))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, new_chat, new_user
|
||||
|
||||
|
||||
class ReplicasTest(unittest.TestCase):
|
||||
def test_one_session_is_served_by_several_replicas(self):
|
||||
"""Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200"""
|
||||
alice = new_user('alice')
|
||||
results = [call('GET', '/api/v1/users/me', token=alice['token']) for _ in range(10)]
|
||||
self.assertEqual({status for status, _, _ in results}, {200})
|
||||
self.assertGreaterEqual(len({headers['X-Instance-Id'] for _, headers, _ in results}), 2)
|
||||
|
||||
def test_write_on_one_replica_is_read_on_another(self):
|
||||
"""Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую"""
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||||
for i in range(6):
|
||||
_, written_by, message = call('POST', path, {'text': 'm%d' % i}, alice['token'])
|
||||
status, read_by, page = call('GET', path + '?limit=100', token=alice['token'])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIn(message['id'], [m['id'] for m in page['items']],
|
||||
'запись через %s не видна через %s' % (written_by['X-Instance-Id'], read_by['X-Instance-Id']))
|
||||
|
||||
def test_every_api_response_has_instance_id(self):
|
||||
"""X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health"""
|
||||
alice = new_user('alice')
|
||||
for method, path, token, expected in (
|
||||
('GET', '/health/live', None, 200), ('GET', '/health/ready', None, 200),
|
||||
('GET', '/api/v1/users/me', None, 401), ('GET', '/api/v1/users/me', alice['token'], 200),
|
||||
('GET', '/api/v1/chats/' + str(uuid.uuid4()), alice['token'], 404), ('GET', '/nope', None, 404)):
|
||||
with self.subTest(path=path, expected=expected):
|
||||
status, headers, _ = call(method, path, token=token)
|
||||
self.assertEqual(status, expected)
|
||||
self.assertTrue(headers['X-Instance-Id'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,81 @@
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from helpers import compose, new_chat, new_user, sql
|
||||
from imagegen import SAMPLE_PNG, noise_png
|
||||
from test_attachments import upload
|
||||
|
||||
LIMITED_CLIENT = '''
|
||||
import json
|
||||
from botocore.exceptions import ClientError
|
||||
from config import settings
|
||||
from s3 import client
|
||||
results = {}
|
||||
for name, action in (
|
||||
("put_get_delete_own_bucket", lambda: (client.put_object(Bucket=settings.s3_bucket, Key="probe/x", Body=b"x"),
|
||||
client.get_object(Bucket=settings.s3_bucket, Key="probe/x"),
|
||||
client.delete_object(Bucket=settings.s3_bucket, Key="probe/x"))),
|
||||
("list_bucket_objects", lambda: client.list_objects_v2(Bucket=settings.s3_bucket)),
|
||||
("create_bucket", lambda: client.create_bucket(Bucket="intruder")),
|
||||
("delete_bucket", lambda: client.delete_bucket(Bucket=settings.s3_bucket)),
|
||||
("list_all_buckets", lambda: client.list_buckets()["Buckets"])):
|
||||
try:
|
||||
value = action()
|
||||
results[name] = "allowed" if name != "list_all_buckets" else "allowed:%d" % len(value)
|
||||
except ClientError as error:
|
||||
results[name] = error.response["Error"]["Code"]
|
||||
print(json.dumps(results))
|
||||
'''
|
||||
|
||||
|
||||
class StorageTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.chat = new_chat(self.alice)
|
||||
|
||||
def object_key(self, attachment_id):
|
||||
return sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment_id)
|
||||
|
||||
def test_object_key_is_generated_by_server(self):
|
||||
"""Ключ объекта = attachments/<uuid вложения>, filename в нём не участвует"""
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG, filename='мой-файл.png')[2]
|
||||
self.assertEqual(self.object_key(att['id']), 'attachments/' + att['id'])
|
||||
|
||||
def test_bucket_is_private_outside_api(self):
|
||||
"""Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403"""
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
for path in ('/messenger-attachments/' + self.object_key(att['id']), '/messenger-attachments'):
|
||||
with self.subTest(path=path):
|
||||
result = compose('exec', '-T', 'lb', 'wget', '-S', '-O', '/dev/null', 'http://s3:8333' + path, check=False)
|
||||
self.assertNotIn('200 OK', result)
|
||||
output = compose('exec', '-T', 'lb', 'sh', '-c', 'wget -S -O /dev/null http://s3:8333%s 2>&1; true' % path)
|
||||
self.assertIn('403', output)
|
||||
|
||||
def test_api_credentials_are_limited(self):
|
||||
"""Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied"""
|
||||
import json
|
||||
results = json.loads(compose('exec', '-T', 'api', 'python', '-c', 'import sys; sys.path.insert(0, "src")\n' + LIMITED_CLIENT))
|
||||
self.assertEqual(results['put_get_delete_own_bucket'], 'allowed')
|
||||
for action in ('list_bucket_objects', 'create_bucket', 'delete_bucket'):
|
||||
self.assertEqual(results[action], 'AccessDenied', results)
|
||||
self.assertNotEqual(results['list_all_buckets'], 'allowed:1', 'API sees the bucket list')
|
||||
|
||||
def test_bootstrap_is_idempotent(self):
|
||||
"""Повторный s3-init: код 0, bucket уже есть, приватность подтверждена"""
|
||||
output = compose('run', '--rm', '--no-deps', 's3-init')
|
||||
self.assertIn('already exists, nothing to do', output)
|
||||
self.assertIn('is private', output)
|
||||
|
||||
def test_no_original_bytes_in_database_or_api_container(self):
|
||||
"""В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто"""
|
||||
binary = sql("SELECT string_agg(column_name, ',') FROM information_schema.columns "
|
||||
"WHERE table_name = 'attachments' AND data_type = 'bytea'")
|
||||
self.assertEqual(binary, 'sha256')
|
||||
self.assertEqual(upload(self.alice, self.chat, noise_png(2 * 1024 * 1024))[0], 201)
|
||||
for container in compose('ps', '-q', 'api').split():
|
||||
files = subprocess.run(['docker', 'exec', container, 'sh', '-c', 'find /tmp -type f | wc -l'],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
self.assertEqual(files, '0', container[:12])
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Usage (стенд запущен; на время эксперимента лучше остановить другие стенды):
|
||||
python student-tests/upload_memory.py --output evidence/upload-memory.json
|
||||
|
||||
N одновременных загрузок PNG ~10 MiB через LB при двух настройках буфера тела запроса
|
||||
(UPLOAD_MEMORY_BYTES = 1 MiB в RAM, дальше временный файл — рабочая настройка; и 16 MiB — весь файл в RAM).
|
||||
Перед каждым прогоном реплики перезапускаются (memory.peak cgroup считается с момента старта контейнера);
|
||||
во время прогона каждые 0,1 с снимаются memory.current и anon/file из memory.stat каждой реплики.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
from helpers import BASE_URL, api_containers, compose, new_chat, new_user, wait_in_rotation, wait_ready
|
||||
from imagegen import noise_png
|
||||
|
||||
MIB = 1024 * 1024
|
||||
SAMPLER = ('while true; do echo "current $(cat /sys/fs/cgroup/memory.current)"; '
|
||||
'grep -E "^(anon|file) " /sys/fs/cgroup/memory.stat; sleep 0.1; done')
|
||||
|
||||
|
||||
class Sampler:
|
||||
def __init__(self, container):
|
||||
self.container, self.peaks = container, {'current': 0, 'anon': 0, 'file': 0}
|
||||
self.process = subprocess.Popen(['docker', 'exec', container, 'sh', '-c', SAMPLER],
|
||||
stdout=subprocess.PIPE, text=True)
|
||||
self.thread = threading.Thread(target=self.read, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def read(self):
|
||||
for line in self.process.stdout:
|
||||
name, _, value = line.partition(' ')
|
||||
if name in self.peaks and value.strip().isdigit():
|
||||
self.peaks[name] = max(self.peaks[name], int(value))
|
||||
|
||||
def stop(self):
|
||||
self.process.terminate()
|
||||
self.process.wait()
|
||||
return {name: round(value / MIB, 1) for name, value in self.peaks.items()}
|
||||
|
||||
|
||||
def cgroup(container, name):
|
||||
return int(subprocess.run(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/' + name],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
|
||||
|
||||
def post(token, chat_id, data):
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=120)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat_id, body=data,
|
||||
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'image/png'})
|
||||
status = connection.getresponse().status
|
||||
except OSError as error:
|
||||
status = type(error).__name__
|
||||
connection.close()
|
||||
return status, time.monotonic() - started
|
||||
|
||||
|
||||
def restart_api(memory_bytes):
|
||||
env = dict(os.environ, UPLOAD_MEMORY_BYTES=str(memory_bytes))
|
||||
compose('up', '-d', '--no-deps', '--no-build', '--force-recreate', 'api', env=env)
|
||||
time.sleep(3)
|
||||
wait_ready()
|
||||
for container in api_containers():
|
||||
wait_in_rotation(container)
|
||||
|
||||
|
||||
def run(label, memory_bytes, parallel, data):
|
||||
restart_api(memory_bytes)
|
||||
alice = new_user('memory')
|
||||
chat = new_chat(alice)
|
||||
replicas = api_containers()
|
||||
idle = {c[:12]: round(cgroup(c, 'memory.current') / MIB, 1) for c in replicas}
|
||||
samplers = [Sampler(c) for c in replicas]
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=parallel) as pool:
|
||||
results = list(pool.map(lambda _: post(alice['token'], chat['id'], data), range(parallel)))
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(0.5)
|
||||
sampled = {c[:12]: s.stop() for c, s in zip(replicas, samplers)}
|
||||
peak = {c[:12]: round(cgroup(c, 'memory.peak') / MIB, 1) for c in replicas}
|
||||
limit = round(cgroup(replicas[0], 'memory.max') / MIB)
|
||||
statuses = [r[0] for r in results]
|
||||
summary = {'label': label, 'upload_memory_bytes': memory_bytes, 'parallel': parallel,
|
||||
'file_mib': round(len(data) / MIB, 2), 'ok': statuses.count(201),
|
||||
'errors': sorted({str(s) for s in statuses if s != 201}), 'wall_s': round(wall, 2),
|
||||
'max_request_s': round(max(r[1] for r in results), 2), 'limit_mib': limit,
|
||||
'idle_current_mib': idle, 'peak_since_start_mib': peak, 'sampled_peak_mib': sampled}
|
||||
worst_anon = max(v['anon'] for v in sampled.values())
|
||||
print('%-28s ok=%2d/%d wall=%5.1fs peak(memory.peak)=%s anon max=%.1f MiB'
|
||||
% (label, summary['ok'], parallel, wall, list(peak.values()), worst_anon))
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output')
|
||||
parser.add_argument('--parallel', default='10,20')
|
||||
args = parser.parse_args()
|
||||
data = noise_png(10 * MIB - 50000)
|
||||
runs = [run('spool 1 MiB, %s parallel' % n, MIB, int(n), data) for n in args.parallel.split(',')]
|
||||
runs.append(run('all in RAM, %s parallel' % args.parallel.split(',')[-1], 16 * MIB,
|
||||
int(args.parallel.split(',')[-1]), data))
|
||||
restart_api(MIB)
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as file:
|
||||
json.dump({'runs': runs}, file, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user