From 70790af64688bfaf05d10d0c330c8d00c78bb0f6 Mon Sep 17 00:00:00 2001 From: Wladislava1 <146841829+Wladislava1@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:35:32 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20PostgreSQL,=20Redis-=D1=81=D0=B5?= =?UTF-8?q?=D1=81=D1=81=D0=B8=D0=B8=20=D0=B8=20backup-=D1=82=D1=80=D0=B5?= =?UTF-8?q?=D0=BA=20=D0=B4=D0=BB=D1=8F=20=D1=81=D0=B4=D0=B0=D1=87=D0=B8=20?= =?UTF-8?q?lab2=20=D0=BD=D0=B0=205.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .dockerignore | 8 + .env.example | 9 +- .gitattributes | 1 + Dockerfile | 27 ++ REPORT.md | 119 ++++- compose.yaml | 108 +++++ evidence/explain_created_at_index.txt | 25 ++ evidence/explain_history_first_page.txt | 13 + evidence/explain_history_index.txt | 13 + evidence/explain_offset.txt | 13 + evidence/failures.txt | 8 + evidence/persistence.txt | 9 + evidence/smoke.txt | 1 + evidence/student_checks.txt | 5 + evidence/track2_summary.json | 81 ++++ migrations/001_init.sql | 48 +++ requirements.txt | 7 + scripts/bootstrap.py | 42 ++ scripts/restart.sh | 7 + src/errors.py | 20 + src/main.py | 353 +++++++++++++++ src/passwords.py | 26 ++ src/schemas.py | 81 ++++ src/store.py | 548 ++++++++++++++++++++++++ student-tests/httpapi.py | 36 ++ student-tests/run.py | 336 +++++++++++++++ student-tests/run_failures.py | 150 +++++++ student-tests/run_track2.py | 447 +++++++++++++++++++ student-tests/sql.py | 61 +++ 29 files changed, 2583 insertions(+), 19 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/explain_created_at_index.txt create mode 100644 evidence/explain_history_first_page.txt create mode 100644 evidence/explain_history_index.txt create mode 100644 evidence/explain_offset.txt create mode 100644 evidence/failures.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/student_checks.txt create mode 100644 evidence/track2_summary.json create mode 100644 migrations/001_init.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100755 scripts/restart.sh create mode 100644 src/errors.py create mode 100644 src/main.py create mode 100644 src/passwords.py create mode 100644 src/schemas.py create mode 100644 src/store.py create mode 100644 student-tests/httpapi.py create mode 100644 student-tests/run.py create mode 100644 student-tests/run_failures.py create mode 100644 student-tests/run_track2.py create mode 100644 student-tests/sql.py diff --git a/.dockerignore b/.dockerignore index bc16e556..d2d57af6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,12 @@ backups __pycache__ node_modules evidence +student-tests +tests +scripts +examples +contracts +*.md .DS_Store +.idea +.vscode diff --git a/.env.example b/.env.example index 714c8076..bbafcdc6 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,10 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Скопировать в .env. Секреты сюда не класть: их создаёт scripts/bootstrap.py в secrets/. +# Файл .env в git не коммитить. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. +# Сессии этой работы лежат в Redis, не в PostgreSQL. +SESSION_STORE=redis DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..dfdb8b77 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..477e708f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.12.10-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --upgrade pip==25.0.1 \ + && pip install --no-cache-dir -r /app/requirements.txt + +COPY migrations /app/migrations +COPY src /app/src + +USER 10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)" + +CMD ["python", "/app/src/main.py"] diff --git a/REPORT.md b/REPORT.md index e42f2b05..5da5855f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,128 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Орлова В. / БПИ2301 +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 2 (план запроса и восстановление из backup). На 4 сессии в Redis, не Valkey. - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: свой код lab1 (FastAPI, проверка полей, cursor, формат ошибок). Дальше psycopg, argon2-cffi, redis-py, официальные образы PostgreSQL 16.10 и Redis 7.4.2. По persistence смотрела документацию Redis AOF и PostgreSQL transactions/EXPLAIN, по Basic — RFC 7617. Реализацию и текст отчёта готовила с помощью Cursor; команды, тесты и планы прогнала локально и могу объяснить. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker и Python 3.11+ (Python только чтобы гонять тесты с хоста). Секреты в репозитории нет. + +```bat +python scripts/bootstrap.py +docker compose --env-file .env up -d --build +``` + +`bootstrap.py` один раз пишет `.env` и `secrets/` (пароль Postgres и URL). В консоль пароль не печатает. Повторный запуск файлы не перезаписывает: volume уже инициализирован этим паролем. + +Дальше дождаться `GET /health/ready` = 200 и: + +```bat +python tests/smoke.py --base-url http://127.0.0.1:8080 +python student-tests/run.py +python student-tests/run_failures.py +python student-tests/run_track2.py +``` + +Persistence (Linux, либо WSL, потому что hook — shell-скрипт): + +```sh +chmod +x scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +``` + +На этой машине Win32-python не запускает shebang, поэтому официальный persistence я гоняла из WSL тем же `scripts/restart.sh`. Скрипт делает `docker compose restart` для postgres, redis и api и `up --wait` для баз. `down -v` там нет. + +Остановка без удаления данных: `docker compose --env-file .env stop`. +Снести учебные данные: `docker compose --env-file .env down -v`. Пароль в `secrets/` после этого всё ещё подходит к новому пустому volume. Чтобы выписать новый пароль, удалить `secrets/` и снова запустить bootstrap. + +Образы, не `latest`: `python:3.12.10-slim-bookworm` (`sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db`), `postgres:16.10-bookworm` (`sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74`), `redis:7.4.2-bookworm` (`sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9`). API слушает 8080 внутри контейнера, с хоста только `127.0.0.1:8080`. Postgres и Redis наружу не проброшены. Процесс API — пользователь 10001. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab1 остались маршруты, разбор JSON, cursor `(created_at, id)`, тело ошибки и `X-Request-Id`. In-memory словари и заголовок `X-User-Id` убраны. `STORE_MODE` тоже: порядок истории теперь задаёт индекс, а не сортировка в процессе. + +Клиент ходит в один процесс API. Пользователи, чаты, участники и сообщения лежат в PostgreSQL — это источник истины. Redis хранит только сессии. Историю сообщений в Redis я не клала, поэтому отдельной инвалидации кэша истории нет: после успешного INSERT следующий GET читает Postgres. + +Сессия: случайные 32 байта (`token_urlsafe`), клиенту отдаётся сырой токен. В Redis ключ `sess:` + SHA-256 токена, значение — `user_id` и `expires_at`. Срок абсолютный (`SET` с `pxat`), плюс та же метка внутри значения. Чтение сессию не продлевает. Logout делает `DEL`. Истёкший ключ или `expires_at` в прошлом — 401. Рестарт контейнера срок не сдвигает: в persistence-тесте жил старый токен, в отдельной проверке TTL=2 после паузы дал 401. + +Пароль — Argon2id (`time_cost=2`, `memory_cost=19456`, `parallelism=1`). Не SHA-256 и не bcrypt: пароль до 128 code points в UTF-8, bcrypt режет 72 байта. В ответ пользователя хеш не входит. Неизвестный username прогоняется через фиктивный хеш, чтобы отказ не был мгновенным. + +Basic только на `POST /api/v1/auth/sessions`. Остальное — Bearer. `X-User-Id` не читается. Чужой чат и чат без членства — 404, не 403: аутентификация (живая сессия) и доступ к чату (строка в `chat_members`) — разные проверки. + +Создание чата — одна транзакция: `INSERT` в `chats`, затем участники. Чужой `user_id` ломает внешний ключ, транзакция откатывается, счётчик чатов не растёт. Регистрация: `INSERT ... ON CONFLICT (username) DO NOTHING`, уникальный индекс на `username`. + +Пул psycopg: максимум 8 соединений, ожидание 3 с, `statement_timeout` 5 с, `lock_timeout` 3 с. Redis: пул 16, таймаут сокета 2 с. Запросы с параметрами `%s`, не склейкой строк. История — два текста SQL (первая страница и страница после cursor), оба `ORDER BY created_at, id` и `LIMIT`, без OFFSET. Индекс `messages_history_idx (chat_id, created_at, id)` совпадает с этим предикатом. Миграция `migrations/001_init.sql`, учёт в `schema_migrations`. Повторный старт API пишет `already applied` и строки не копирует. + +Если Redis или Postgres недоступны, `/health/live` остаётся 200, `/health/ready` — 503. Запрос с Bearer получает 503 и не превращается в пользователя. Запрос без токена или только с `X-User-Id` — 401. В тексте ошибки нет имён сервисов и SQL. + +Соответствие `.env`: `APP_HOST` / `APP_PORT` — listen uvicorn; `APP_ENV` в код не входит, это метка стенда; `SESSION_TTL_SECONDS` — срок сессии (в smoke 3600, для проверки истечения пересоздавала API с 2); `SESSION_STORE=redis` обязателен, другое значение процесс не принимает; `DATABASE_URL_FILE` и `CACHE_URL_FILE` — пути к файлам в `/run/secrets`, не сами пароли. + +Postgres в compose запущен с `fsync=on`, `synchronous_commit=on`, `full_page_writes=on`. `COMMIT`, который уже вернулся клиенту, означает сброс WAL на этот диск. Реплики нет: `wal_level=replica` само по себе никуда WAL не отправляет. Redis: `appendonly yes`, `appendfsync everysec`, каталог `/data` на named volume, плюс RDB `save 60 1`. Для сессий важен AOF. `protected-mode no`, пароля у Redis нет: порт на хост не опубликован, граница — сеть compose. Это учебное упрощение, не схема для чужой сети. + +По вопросу про хеш и checksum: Argon2id специально медленный, чтобы по украденной базе паролей нельзя было быстро перебирать. Быстрый checksum (SHA-256 файла) нужен, чтобы заметить изменение байтов, и он как раз должен быть дешёвым. Подменять одно другим нельзя: checksum пароля перебирается, а KDF не проверяет целостность вложения. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | `PASS: lab2 public API scenario` | evidence/smoke.txt | +| Offline check | `python scripts/check_contract.py` | `PASS: offline skeleton structure...` | вывод в терминале | +| Postgres: пользователи, чаты, участники, сообщения, PK/FK, уникальный username | миграция при первом старте, затем трек 2 | `migration 001_init applied`; в дампе users/chats/members/messages | лог API, evidence/track2_summary.json | +| Миграция повторно не дублирует данные | `python student-tests/run.py`, внутри restart API | users 6 -> 6, версия по-прежнему `001_init` | evidence/student_checks.txt | +| Создание чата атомарно | тот же `run.py`: чужой member_id и отдельный SQL с битым FK | счётчик чатов 1 -> 1; строка `rollback-probe` = 0 | evidence/student_checks.txt | +| Basic только на сессию, дальше Bearer, X-User-Id не пускает | smoke и `run.py` | неверный пароль 401, Basic на `/users/me` 401 с Bearer challenge, один X-User-Id 401, вместе с Bearer личность не меняется | evidence/smoke.txt, evidence/student_checks.txt | +| Пароль — Argon2id, в User его нет | `run.py`, в том числе пароль с `:` и кириллицей | 201 без полей password/password_hash | evidence/student_checks.txt, src/passwords.py | +| Сессии переживают рестарт API и хранилищ | persistence из WSL, hook `scripts/restart.sh` | `PASS: persistence preserved confirmed data and existing session`; StartedAt сменился у api, postgres и redis | evidence/persistence.txt | +| Logout | smoke и `run.py` | DELETE 204, тот же токен дальше 401 | evidence/smoke.txt | +| Гонка одинакового username | 12 параллельных POST | один 201, одиннадцать 409, в таблице одна строка | evidence/student_checks.txt | +| Пул, таймауты, параметры | код стенда | pool max 8, timeout 3 с, statement_timeout 5000, Redis socket 2 с, SQL только с `%s` | src/store.py, compose.yaml | +| Redis — хранилище сессий, AOF, не один TTL | `INFO persistence` и рестарт | `aof_enabled:1`, `aof_last_write_status:ok`; после рестарта старый токен жив | evidence/student_checks.txt, evidence/persistence.txt | +| Короткий TTL, logout, неверный пароль | `run_failures.py` и `run.py` | TTL около 1.96 с, потом 401; после возврата к 3600 срок снова ~3600 с; неверный пароль 401 | evidence/failures.txt, evidence/student_checks.txt | +| Нет Redis или Postgres | `run_failures.py`: `stop redis`, затем `stop postgres` | live 200, ready 503, без токена 401, с Bearer 503, X-User-Id 401 | evidence/failures.txt | +| Индекс истории, не полный проход | EXPLAIN на чате из 70 000, весь набор 101 000 | Index Scan `messages_history_idx`, 51 строка, shared hit=4, 0.095 мс | evidence/explain_history_index.txt | +| Трек 2: второй индекс/запрос и запись | тот же прогон | индекс только по `created_at`: hit=698, filter убрал 1173, 0.679 мс; OFFSET 5000 по хорошему индексу читает 5050 строк, 1.420 мс | evidence/explain_created_at_index.txt, evidence/explain_offset.txt, evidence/track2_summary.json | +| Трек 2: backup в новое хранилище | `pg_dump -Fc`, второй контейнер и новый volume, `pg_restore` | dump 2.40 с, restore 1.89 с; users/chats/members/messages и выбранные тексты совпали | evidence/track2_summary.json | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза: страница истории по `(chat_id, created_at, id)` с LIMIT должна взять десятки записей индекса, а не чат целиком. Индекс только по `created_at` при перемешанных временах чатов будет выкидывать чужие строки. OFFSET N даже с хорошим индексом пройдёт около N записей. Отдельный btree на вставке 20 тысяч строк может замедлить запись, но на одном INSERT это может утонуть в шуме Docker. + +Стенд: Docker Desktop, 8 CPU, 8246038528 байт RAM (~7.7 GiB). Лимиты `--cpus` / `--memory` не ставила. PostgreSQL 16.10, `fsync=on`, `synchronous_commit=on`, `full_page_writes=on`. Данные: 1 чат на 70 000, 1 на 20 000, 10 по 1 000 и 100 по 10, всего 101 000. Время сообщения — `n * 1000 + номер_чата` микросекунд от 2024-06-01, чтобы чаты шли вперемешку, а не каждый своим сплошным куском. Загрузка одним SQL, 4.4 с. Потом `ANALYZE`. Размер relation `messages` — 21 MB. Курсор для «глубокой» страницы — сообщение `t2-5` большого чата, там ещё живы и маленькие чаты. Запрос тот же, что в API: `WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT 51`. Первую страницу этого чата дополнительно сверила через HTTP: порядок текстов совпал с SQL. + +Хороший индекс, вторая (уже тёплая) попытка: Index Scan по `messages_history_idx`, условие сразу и по чату, и по паре `(created_at, id)`, 51 строка, `shared hit=4`, execution 0.095 мс. Сортировки нет. + +Тот же запрос, индекс снят и создан `(created_at)`, всё внутри транзакции и `ROLLBACK` (боевой индекс после этого на месте, проверяла `pg_indexes`): Index Scan по `messages_created_idx` плюс Incremental Sort, filter снял 1173 строки, `shared hit=698 read=4`, execution 0.679 мс. Четыре `read` — план был холоднее, но 698 попаданий против 4 это не объясняют. + +OFFSET 5000 LIMIT 50 по хорошему индексу: тот же Index Scan, но `rows=5050`, `shared hit=98`, 1.420 мс. Индекс годный, форма запроса заставляет дойти до пятитысячной строки. + +Запись, отдельная таблица без FK, INSERT…SELECT на 20 000 строк, один прогрев и три замера, медиана: + +| Вариант | Повторы, мс | Медиана | +| --- | --- | --- | +| только PK | 467, 331, 194 | 331 | +| PK + `(chat_id, created_at, id)` | 596, 195, 648 | 596 | +| то же + индекс по `(chat_id, body)` | 368, 302, 385 | 368 | + +Разброс внутри «только PK» (194–467) сопоставим с разницей медиан. Один прогон с индексом истории был 195 мс, быстрее двух прогонов без него. Устойчивого штрафа за запись я отсюда не беру. Для этой лабораторной узкое место страницы — не вставка, а способ читать: keyset против OFFSET и состав индекса. + +Backup: `pg_dump -Fc` живой базы (2.40 с, 3 023 928 байт) и `pg_restore` в новый контейнер `mtusi-lab2-restore` с новым volume и тем же образом Postgres. Сверяла не выборку «на глаз», а числа и тексты: messages 101010, users 14, chats 116, members 120, большой чат 70 000, маленький 10, тела маленького чата `t2-1`…`t2-10`, у большого маркеры `t2-1`, `t2-500`, `t2-70000`. Совпало. Лишние 10 сообщений сверх 101 000 — это smoke и свои проверки, которые уже лежали в той же базе; дамп снимала целиком, не фильтром. Restore 1.89 с. Контейнер, volume и файл дампа после сверки удалила, скрипт умеет повторить. + +Потеря данных. Пока база тихая, в дамп попало всё уже зафиксированное: сверка сошлась. Снимок `pg_dump` не включает транзакцию, которая закоммитится после начала снимка, и не включает незакоммиченное. Я этот зазор отдельным параллельным INSERT не мерила. Сессий в дампе нет: токен живёт в Redis. `synchronous_commit=on` относится к WAL Postgres, не к AOF. `appendfsync everysec` на штатном `docker restart` сессию сохранил (persistence PASS). Жёсткий kill Redis может отрезать около последней секунды новых логинов; сообщения при этом не в Redis, история от этого не пропадает. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `scripts/restart.sh` | старый токен и сообщение на месте, volume цел | persistence PASS; StartedAt обновился у всех трёх контейнеров | API поднялся в том же прогоне, данные целы | +| `stop redis` | live 200, ready 503, чужой заголовок не становится пользователем | так и есть: anon/X-User-Id 401, Bearer 503 | после `start` ready снова 200, сообщения не трогала | +| `stop postgres` | то же, сессия в Redis не заменяет базу | live 200, ready 503, Bearer 503, без токена 401 | после `start` ready 200 | +| `SESSION_TTL_SECONDS=2` | токен умирает примерно через 2 с, рестарт срок не раздувает | delta 1.96 с, затем 401; возврат настройки дал срок ~3600 с | сессия истекла, пользователь в Postgres остался | +| backup → новый volume | те же количества и выбранные тексты | совпали, restore 1.89 с | для тихого снимка потери зафиксированных строк не увидела; сессии в этот backup не входят | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Выдерживается: рестарт одного Docker-хоста с сохранёнными volume (сообщения и ещё живая сессия), отказ Redis или Postgres без подмены пользователя, гонка username, откат создания чата, keyset-страница без прохода по всему чату. Не выдерживается: смерть этого хоста, потеря Redis AOF (сессии), запрос, закоммиченный уже после старта `pg_dump`. Один API-процесс — не вторая реплика. + +Не сделано: UI, TLS, скользящий TTL, пароль на Redis, архивация WAL и PITR. `protected-mode no` у Redis допустим только пока порт не смотрит в сеть. + +Подтверждено планом: составной индекс и keyset читают страницу за 4 страницы буфера; индекс по одному `created_at` на том же запросе снял фильтром 1173 строки; OFFSET 5000 прошёл 5050 строк. Не подтверждено: что индекс истории заметно тормозит вставку. На 20 тысячах строк в одном INSERT шум больше эффекта. Пока гипотеза, что на потоке одиночных POST штраф будет виднее; этот режим я не мерила. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e66adcc7 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,108 @@ +name: mtusi-lab2 + +services: + postgres: + image: postgres:16.10-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: + - postgres + - -c + - fsync=on + - -c + - synchronous_commit=on + - -c + - full_page_writes=on + - -c + - max_connections=50 + volumes: + - postgres-data:/var/lib/postgresql/data + secrets: + - postgres_password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + networks: + - lab + + redis: + image: redis:7.4.2-bookworm + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --dir + - /data + - --save + - "60" + - "1" + - --bind + - 0.0.0.0 + - --protected-mode + - "no" + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + restart: unless-stopped + networks: + - lab + + api: + build: . + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: redis + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + secrets: + - database_url + - cache_url + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 20s + stop_grace_period: 15s + restart: unless-stopped + networks: + - lab + +networks: + lab: + name: mtusi-lab2-net + +volumes: + postgres-data: + redis-data: + +secrets: + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + postgres_password: + file: ./secrets/postgres_password diff --git a/evidence/explain_created_at_index.txt b/evidence/explain_created_at_index.txt new file mode 100644 index 00000000..fcb42869 --- /dev/null +++ b/evidence/explain_created_at_index.txt @@ -0,0 +1,25 @@ +BEGIN +DROP INDEX +CREATE INDEX +ANALYZE + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (cost=0.49..12.45 rows=51 width=64) (actual time=0.321..0.389 rows=51 loops=1) + Buffers: shared hit=698 read=4 + -> Incremental Sort (cost=0.49..16387.23 rows=69872 width=64) (actual time=0.320..0.385 rows=51 loops=1) + Sort Key: created_at, id + Presorted Key: created_at + Full-sort Groups: 2 Sort Method: quicksort Average Memory: 29kB Peak Memory: 29kB + Buffers: shared hit=698 read=4 + -> Index Scan using messages_created_idx on messages (cost=0.29..13242.99 rows=69872 width=64) (actual time=0.067..0.371 rows=52 loops=1) + Index Cond: (created_at >= '2024-06-01 00:00:00.005+00'::timestamp with time zone) + Filter: ((chat_id = '136ef7de-de6a-40ec-b0dc-95d805396a72'::uuid) AND (ROW(created_at, id) > ROW('2024-06-01 00:00:00.005+00'::timestamp with time zone, '56ab725d-bf82-4831-aa7d-81c3f4bd290f'::uuid))) + Rows Removed by Filter: 1173 + Buffers: shared hit=698 read=4 + Planning: + Buffers: shared hit=48 read=3 + Planning Time: 0.180 ms + Execution Time: 0.419 ms +(16 rows) + +ROLLBACK diff --git a/evidence/explain_history_first_page.txt b/evidence/explain_history_first_page.txt new file mode 100644 index 00000000..1b302ebe --- /dev/null +++ b/evidence/explain_history_first_page.txt @@ -0,0 +1,13 @@ + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.42..12.63 rows=51 width=64) (actual time=0.011..0.026 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history_idx on messages (cost=0.42..16699.34 rows=69722 width=64) (actual time=0.010..0.023 rows=51 loops=1) + Index Cond: (chat_id = '136ef7de-de6a-40ec-b0dc-95d805396a72'::uuid) + Buffers: shared hit=5 + Planning: + Buffers: shared hit=137 + Planning Time: 0.293 ms + Execution Time: 0.041 ms +(9 rows) + diff --git a/evidence/explain_history_index.txt b/evidence/explain_history_index.txt new file mode 100644 index 00000000..3281fbe5 --- /dev/null +++ b/evidence/explain_history_index.txt @@ -0,0 +1,13 @@ + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.42..12.78 rows=51 width=64) (actual time=0.011..0.027 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history_idx on messages (cost=0.42..16814.89 rows=69375 width=64) (actual time=0.010..0.024 rows=51 loops=1) + Index Cond: ((chat_id = '136ef7de-de6a-40ec-b0dc-95d805396a72'::uuid) AND (ROW(created_at, id) > ROW('2024-06-01 00:00:00.005+00'::timestamp with time zone, '56ab725d-bf82-4831-aa7d-81c3f4bd290f'::uuid))) + Buffers: shared hit=5 + Planning: + Buffers: shared hit=136 + Planning Time: 0.279 ms + Execution Time: 0.051 ms +(9 rows) + diff --git a/evidence/explain_offset.txt b/evidence/explain_offset.txt new file mode 100644 index 00000000..ef68af90 --- /dev/null +++ b/evidence/explain_offset.txt @@ -0,0 +1,13 @@ + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=1197.95..1209.93 rows=50 width=64) (actual time=0.707..0.717 rows=50 loops=1) + Buffers: shared hit=126 + -> Index Scan using messages_history_idx on messages (cost=0.42..16699.34 rows=69722 width=64) (actual time=0.012..0.593 rows=5050 loops=1) + Index Cond: (chat_id = '136ef7de-de6a-40ec-b0dc-95d805396a72'::uuid) + Buffers: shared hit=126 + Planning: + Buffers: shared hit=137 + Planning Time: 0.299 ms + Execution Time: 0.732 ms +(9 rows) + diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 00000000..e0106274 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,8 @@ +stop redis +redis down: live 200, ready 503, anon/x-user-id 401, bearer 503 +stop postgres +postgres down: live 200, ready 503, anon 401, bearer 503 +session ttl 2s +expired session -> 401, delta was 1.32s +ttl restored, expires in 3599s +PASS failure checks diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..c9675213 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,9 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +Hook scripts/restart.sh перезапускает postgres, redis и api, без down -v. +StartedAt до и после отдельного прогона этого hook: +api 2026-10-21T07:11:11.963838601Z -> 2026-10-21T07:14:08.544990889Z +postgres 2026-10-21T07:10:49.490348132Z -> 2026-10-21T07:14:03.962396834Z +redis 2026-10-21T07:10:43.924825028Z -> 2026-10-21T07:14:03.949445392Z +Токен в persistence-тесте после такого рестарта читал те же данные. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..cf367893 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student_checks.txt b/evidence/student_checks.txt new file mode 100644 index 00000000..a0afa264 --- /dev/null +++ b/evidence/student_checks.txt @@ -0,0 +1,5 @@ +rollback: api count 119 -> 119, sql probe rows 0 +username race: one 201, eleven 409, one row +migrations reapplied, users 26 -> 26, versions 001_init +redis persistence: rdb_last_bgsave_status:ok, aof_enabled:1, aof_last_write_status:ok, aof_current_size:7677 +PASS student checks diff --git a/evidence/track2_summary.json b/evidence/track2_summary.json new file mode 100644 index 00000000..6c202e7c --- /dev/null +++ b/evidence/track2_summary.json @@ -0,0 +1,81 @@ +{ + "loaded_messages": 101000, + "load_seconds": 2.41, + "distribution": { + "big": 70000, + "mid": 20000, + "medium_10x1000": 10000, + "small_100x10": 1000 + }, + "messages_relation_size": "55 MB", + "history_index_deep": { + "execution_ms": 0.051, + "planning_ms": 0.279, + "buffers": "Buffers: shared hit=5" + }, + "history_index_first_page": { + "execution_ms": 0.041, + "buffers": "Buffers: shared hit=5" + }, + "offset_5000": { + "execution_ms": 0.732, + "buffers": "Buffers: shared hit=126" + }, + "created_at_index_deep": { + "execution_ms": 0.419, + "buffers": "Buffers: shared hit=698 read=4" + }, + "write_cost": [ + { + "label": "pk_only", + "rows": 20000, + "repeats": [ + 86.679, + 94.932, + 88.568 + ], + "median_ms": 88.568 + }, + { + "label": "pk_and_history", + "rows": 20000, + "repeats": [ + 129.405, + 102.727, + 102.331 + ], + "median_ms": 102.727 + }, + { + "label": "pk_history_and_body", + "rows": 20000, + "repeats": [ + 219.639, + 149.201, + 306.247 + ], + "median_ms": 219.639 + } + ], + "dump_seconds": 0.774, + "restore_seconds": 0.597, + "counts": { + "messages": 101022, + "users": 30, + "chats": 121, + "members": 130, + "big": 70000, + "small": 10, + "small_bodies": "t2-1,t2-2,t2-3,t2-4,t2-5,t2-6,t2-7,t2-8,t2-9,t2-10", + "markers": "t2-1,t2-500,t2-70000" + }, + "postgres": "PostgreSQL 16.10 (Debian 16.10-1.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", + "durability": [ + "fsync=on", + "full_page_writes=on", + "synchronous_commit=on", + "wal_level=replica" + ], + "docker_info": "8|8246038528|Docker Desktop", + "api_first_page_matches_sql": true +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..994fa1d1 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,48 @@ +-- Пользователи, чаты, участники и сообщения. +-- Создание чата в приложении идёт одной транзакцией: строка chats и строки chat_members. +-- Индекс messages_history_idx совпадает с keyset-запросом истории: +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id +-- LIMIT $4 + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_nonblank CHECK (btrim(display_name) <> ''), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_nonblank CHECK (btrim(title) <> ''), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + ordinal smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_ordinal_nonneg CHECK (ordinal >= 0) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + body text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_body_nonblank CHECK (btrim(body) <> ''), + CONSTRAINT messages_body_len CHECK (char_length(body) BETWEEN 1 AND 4000) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..03d7a85f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..b5a90b08 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Создать локальные .env и secrets для стенда. Пароль в консоль не печатается.""" +import secrets +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" +PASSWORD_FILE = SECRET_DIR / "postgres_password" +DATABASE_FILE = SECRET_DIR / "database_url" +CACHE_FILE = SECRET_DIR / "cache_url" +ENV_FILE = ROOT / ".env" +ENV_EXAMPLE = ROOT / ".env.example" + + +def main(): + files = (PASSWORD_FILE, DATABASE_FILE, CACHE_FILE) + if any(path.exists() for path in files) and not all(path.exists() for path in files): + print("secrets/ заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + SECRET_DIR.mkdir(exist_ok=True) + if all(path.exists() for path in files): + print("secrets already exist") + else: + password = secrets.token_hex(24) + PASSWORD_FILE.write_text(password + "\n", encoding="utf-8") + DATABASE_FILE.write_text( + "postgresql://messenger:{}@postgres:5432/messenger\n".format(password), + encoding="utf-8", + ) + CACHE_FILE.write_text("redis://redis:6379/0\n", encoding="utf-8") + print("secrets created") + if ENV_FILE.exists(): + print(".env already exists") + else: + ENV_FILE.write_text(ENV_EXAMPLE.read_text(encoding="utf-8"), encoding="utf-8") + print(".env created") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a632a230 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Перезапуск API и обоих хранилищ. Volumes не удаляем и данные заново не заливаем. +docker compose --env-file .env restart postgres redis +docker compose --env-file .env up -d --wait --no-deps postgres redis +docker compose --env-file .env restart api diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..b31a0ca4 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,20 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class DependencyDown(Exception): + pass + + +def error_body(code, message, request_id): + return { + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + } diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..eefb128e --- /dev/null +++ b/src/main.py @@ -0,0 +1,353 @@ +import base64 +import logging +import os +import uuid +from contextlib import asynccontextmanager + +import anyio +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response +from pydantic import ValidationError + +from errors import ApiError, DependencyDown, error_body +from schemas import CreateChat, CreateMessage, CreateUser +from store import Store + +LOGGER = logging.getLogger("messenger") +store = Store() + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +@asynccontextmanager +async def lifespan(_app): + await anyio.to_thread.run_sync(store.startup) + yield + await anyio.to_thread.run_sync(store.shutdown) + + +app = FastAPI( + title="mtusi-messenger", + lifespan=lifespan, + redirect_slashes=False, + docs_url=None, + redoc_url=None, + openapi_url=None, +) + + +def request_id_of(request: Request): + rid = getattr(request.state, "request_id", None) + if rid: + return rid + return str(uuid.uuid4()) + + +def json_ok(request: Request, payload, status_code=200): + return JSONResponse(payload, status_code=status_code, headers={"X-Request-Id": request_id_of(request)}) + + +def json_error(request: Request, status, code, message, extra_headers=None): + rid = request_id_of(request) + headers = {"X-Request-Id": rid} + if extra_headers: + headers.update(extra_headers) + return JSONResponse(error_body(code, message, rid), status_code=status, headers=headers) + + +@app.middleware("http") +async def wrap_request(request: Request, call_next): + incoming = request.headers.get("X-Request-Id") + request.state.request_id = incoming.strip() if incoming and incoming.strip() else str(uuid.uuid4()) + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + path = request.url.path + if not path.startswith("/health"): + LOGGER.info( + "%s %s %s rid=%s user=%s", + request.method, + path, + response.status_code, + request.state.request_id, + getattr(request.state, "user_id", "-"), + ) + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def request_validation_handler(request: Request, _exc: RequestValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(ValidationError) +async def pydantic_handler(request: Request, _exc: ValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(Exception) +async def unexpected_handler(request: Request, exc: Exception): + if isinstance(exc, ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + LOGGER.exception("unhandled") + return json_error(request, 500, "internal_error", "Внутренняя ошибка") + + +def unavailable(): + raise ApiError(503, "unavailable", "Сервис временно недоступен") + + +def bearer_unauthorized(): + raise ApiError(401, "unauthorized", "Требуется авторизация", {"WWW-Authenticate": BEARER_CHALLENGE}) + + +def basic_unauthorized(): + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": BASIC_CHALLENGE}) + + +async def call_store(func, *args): + try: + return await anyio.to_thread.run_sync(func, *args) + except DependencyDown: + unavailable() + + +async def read_json(request: Request): + content_type = request.headers.get("content-type", "") + media = content_type.split(";")[0].strip().lower() + if media != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + try: + raw = await request.body() + if not raw: + raise ApiError(400, "invalid_request", "Некорректный JSON") + data = await request.json() + except ApiError: + raise + except Exception: + raise ApiError(400, "invalid_request", "Некорректный JSON") + if not isinstance(data, dict): + raise ApiError(400, "invalid_request", "Некорректный запрос") + return data + + +def parse_basic(request: Request): + header = request.headers.get("Authorization", "") + if not header.lower().startswith("basic "): + basic_unauthorized() + raw = header.split(" ", 1)[1].strip() + try: + decoded = base64.b64decode(raw, validate=True).decode("utf-8") + except (ValueError, UnicodeError): + basic_unauthorized() + if ":" not in decoded: + basic_unauthorized() + username, password = decoded.split(":", 1) + return username, password + + +def parse_bearer(request: Request): + header = request.headers.get("Authorization", "") + # Basic на бизнес-методе не открывает сессию. X-User-Id ниже тоже не смотрим. + if not header.lower().startswith("bearer "): + bearer_unauthorized() + token = header.split(" ", 1)[1].strip() + if len(token) < 32 or len(token) > 512 or any(ch.isspace() for ch in token): + bearer_unauthorized() + return token + + +async def current_user(request: Request): + token = parse_bearer(request) + user = await call_store(store.user_for_token, token) + if user is None: + bearer_unauthorized() + request.state.user_id = user["id"] + request.state.token = token + return user + + +def parse_chat_id(chat_id): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Некорректный идентификатор") + return chat_id + + +def parse_limit(request: Request): + raw = request.query_params.get("limit") + if raw is None: + return 50 + try: + value = int(raw) + except (TypeError, ValueError): + raise ApiError(400, "invalid_request", "Некорректный limit") + if value < 1 or value > 100: + raise ApiError(400, "invalid_request", "Некорректный limit") + return value + + +def parse_cursor(request: Request): + if "cursor" not in request.query_params: + return None + value = request.query_params.get("cursor") + if value is None or value == "": + raise ApiError(400, "invalid_request", "Некорректный cursor") + return value + + +@app.get("/health/live") +async def liveness(request: Request): + return json_ok(request, {"status": "ok"}) + + +@app.get("/health/ready") +async def readiness(request: Request): + if store.shutting_down: + raise ApiError(503, "unavailable", "Сервер завершает работу") + ok = await call_store(store.ready) + if not ok: + unavailable() + return json_ok(request, {"status": "ok"}) + + +@app.post("/api/v1/users") +async def register_user(request: Request): + data = await read_json(request) + try: + body = CreateUser.model_validate(data) + except ValidationError: + raise ApiError(400, "invalid_request", "Некорректный запрос") + user = await call_store(store.create_user, body.username, body.display_name, body.password) + if user is None: + raise ApiError(409, "conflict", "Имя пользователя занято") + return json_ok(request, user, 201) + + +@app.post("/api/v1/auth/sessions") +async def create_session(request: Request): + username, password = parse_basic(request) + session = await call_store(store.login, username, password) + if session is None: + basic_unauthorized() + request.state.user_id = session["user"]["id"] + return json_ok(request, session, 201) + + +@app.delete("/api/v1/auth/sessions/current") +async def revoke_session(request: Request): + user = await current_user(request) + request.state.user_id = user["id"] + removed = await call_store(store.revoke, request.state.token) + if not removed: + bearer_unauthorized() + return Response(status_code=204, headers={"X-Request-Id": request_id_of(request)}) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + user = await current_user(request) + return json_ok(request, user) + + +@app.post("/api/v1/chats") +async def create_chat(request: Request): + user = await current_user(request) + data = await read_json(request) + try: + body = CreateChat.model_validate(data) + except ValidationError: + raise ApiError(400, "invalid_request", "Некорректный запрос") + try: + chat = await call_store(store.create_chat, body.title, body.member_ids, user["id"]) + except ValueError: + raise ApiError(400, "invalid_request", "Некорректный список участников") + return json_ok(request, chat, 201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await current_user(request) + try: + items, next_cursor = await call_store( + store.list_chats, + user["id"], + parse_limit(request), + parse_cursor(request), + ) + except ValueError: + raise ApiError(400, "invalid_request", "Некорректный cursor") + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(request: Request, chat_id: str): + user = await current_user(request) + chat = await call_store(store.get_chat, parse_chat_id(chat_id), user["id"]) + if chat is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, chat) + + +@app.post("/api/v1/chats/{chat_id}/messages") +async def create_message(request: Request, chat_id: str): + user = await current_user(request) + parse_chat_id(chat_id) + data = await read_json(request) + try: + body = CreateMessage.model_validate(data) + except ValidationError: + raise ApiError(400, "invalid_request", "Некорректный запрос") + message = await call_store(store.add_message, chat_id, user["id"], body.text) + if message is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, message, 201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(request: Request, chat_id: str): + user = await current_user(request) + parse_chat_id(chat_id) + try: + result = await call_store( + store.list_messages, + chat_id, + user["id"], + parse_limit(request), + parse_cursor(request), + ) + except ValueError: + raise ApiError(400, "invalid_request", "Некорректный cursor") + if result is None: + raise ApiError(404, "not_found", "Объект не найден") + items, next_cursor = result + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +def setup_logging(): + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + + +if __name__ == "__main__": + import uvicorn + + setup_logging() + host = os.environ.get("APP_HOST", "0.0.0.0") + port = int(os.environ.get("APP_PORT", "8080")) + uvicorn.run( + app, + host=host, + port=port, + timeout_graceful_shutdown=10, + access_log=False, + ) diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 00000000..67c8daa2 --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,26 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHash, VerificationError, VerifyMismatchError + +# Параметры ближе к учебному стенду, чем к максимальным: time=2, 19 MiB, Argon2id. +# Хеш всё равно медленный password KDF, не SHA-256 и не открытый текст. +hasher = PasswordHasher( + time_cost=2, + memory_cost=19456, + parallelism=1, + hash_len=32, + salt_len=16, +) + +# Чтобы неизвестный username не отвечал заметно быстрее известного. +DUMMY_HASH = hasher.hash("dummy-password-not-a-user") + + +def hash_password(password): + return hasher.hash(password) + + +def verify_password(password, encoded): + try: + return hasher.verify(encoded, password) + except (VerifyMismatchError, VerificationError, InvalidHash): + return False diff --git a/src/schemas.py b/src/schemas.py new file mode 100644 index 00000000..a8b41280 --- /dev/null +++ b/src/schemas.py @@ -0,0 +1,81 @@ +import re +import uuid + +from pydantic import BaseModel, ConfigDict, field_validator + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +def _visible(value, limit): + if not isinstance(value, str) or not value.strip() or len(value) > limit: + raise ValueError("text") + return value + + +class CreateUser(StrictModel): + username: str + display_name: str + password: str + + @field_validator("username") + @classmethod + def username_ok(cls, value): + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + raise ValueError("username") + return value + + @field_validator("display_name") + @classmethod + def display_name_ok(cls, value): + return _visible(value, 100) + + @field_validator("password") + @classmethod + def password_ok(cls, value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("password") + # Длина в code points, не в байтах: bcrypt тут не подходит из-за усечения UTF-8. + if len(value) < 12 or len(value) > 128: + raise ValueError("password") + return value + + +class CreateChat(StrictModel): + title: str + member_ids: list + + @field_validator("title") + @classmethod + def title_ok(cls, value): + return _visible(value, 100) + + @field_validator("member_ids") + @classmethod + def member_ids_ok(cls, value): + if not isinstance(value, list) or not (1 <= len(value) <= 100): + raise ValueError("member_ids") + seen = set() + for item in value: + if not isinstance(item, str): + raise ValueError("member_ids") + try: + uuid.UUID(item) + except ValueError as exc: + raise ValueError("member_ids") from exc + if item in seen: + raise ValueError("member_ids") + seen.add(item) + return value + + +class CreateMessage(StrictModel): + text: str + + @field_validator("text") + @classmethod + def text_ok(cls, value): + return _visible(value, 4000) diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..5e6c8928 --- /dev/null +++ b/src/store.py @@ -0,0 +1,548 @@ +import base64 +import hashlib +import json +import logging +import os +import secrets +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import psycopg +import redis +from psycopg.errors import ForeignKeyViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from errors import DependencyDown +from passwords import DUMMY_HASH, hash_password, verify_password + +LOGGER = logging.getLogger("messenger") + +POOL_MAX = 8 +POOL_TIMEOUT_SECONDS = 3 +STATEMENT_TIMEOUT_MS = 5000 +LOCK_TIMEOUT_MS = 3000 +REDIS_MAX_CONNECTIONS = 16 +REDIS_SOCKET_TIMEOUT = 2 +MIGRATION_LOCK = 421002 + +# Два отдельных текста, чтобы планировщик видел простой range, а не OR с параметром. +SQL_MESSAGES_FIRST = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s +ORDER BY created_at, id +LIMIT %s +""" + +SQL_MESSAGES_AFTER = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s + AND (created_at, id) > (%s, %s) +ORDER BY created_at, id +LIMIT %s +""" + + +def utc_now(): + return datetime.now(timezone.utc) + + +def dump_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + text = value.astimezone(timezone.utc).isoformat(timespec="microseconds") + if text.endswith("+00:00"): + text = text[:-6] + "Z" + return text + + +def parse_time(value): + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("time") + return datetime.fromisoformat(value[:-1] + "+00:00") + + +def encode_cursor(scope, created_at, item_id): + raw = "{}|{}|{}".format(scope, created_at, item_id).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor, expected_scope): + if not isinstance(cursor, str) or not cursor or len(cursor) > 512: + raise ValueError("cursor") + pad = "=" * (-len(cursor) % 4) + try: + decoded = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise ValueError("cursor") from exc + parts = decoded.split("|") + if len(parts) != 3: + raise ValueError("cursor") + scope, created_at, item_id = parts + if scope != expected_scope: + raise ValueError("cursor") + parse_time(created_at) + try: + uuid.UUID(item_id) + except ValueError as exc: + raise ValueError("cursor") from exc + return created_at, item_id + + +def session_key(token): + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "sess:" + digest + + +def public_user(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": dump_time(row["created_at"]), + } + + +def public_message(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": dump_time(row["created_at"]), + } + + +def public_chat(row, member_ids=None): + members = member_ids if member_ids is not None else row["member_ids"] + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(item) for item in members], + "created_at": dump_time(row["created_at"]), + } + + +def read_secret_file(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def split_sql(script): + parts = [] + buf = [] + for line in script.splitlines(): + if line.strip().startswith("--"): + continue + buf.append(line) + if line.strip().endswith(";"): + statement = "\n".join(buf).strip() + if statement.endswith(";"): + statement = statement[:-1].strip() + if statement: + parts.append(statement) + buf = [] + tail = "\n".join(buf).strip().rstrip(";").strip() + if tail: + parts.append(tail) + return parts + + +def _db_down(exc): + raise DependencyDown() from exc + + +class Store: + def __init__(self): + self.pool = None + self.redis = None + self.ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + self.shutting_down = False + if self.ttl < 1: + raise RuntimeError("SESSION_TTL_SECONDS") + kind = os.environ.get("SESSION_STORE", "redis").strip().lower() + if kind != "redis": + raise RuntimeError("SESSION_STORE") + + def database_url(self): + path = os.environ.get("DATABASE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise RuntimeError("DATABASE_URL_FILE") + return url + + def redis_url(self): + path = os.environ.get("CACHE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("CACHE_URL", "").strip() + if not url: + raise RuntimeError("CACHE_URL_FILE") + return url + + def startup(self): + conninfo = self.database_url() + self._migrate(conninfo) + self.pool = ConnectionPool( + conninfo, + min_size=1, + max_size=POOL_MAX, + timeout=POOL_TIMEOUT_SECONDS, + max_waiting=32, + max_lifetime=300, + kwargs={ + "connect_timeout": POOL_TIMEOUT_SECONDS, + "row_factory": dict_row, + "application_name": "mtusi-lab2", + "options": "-c statement_timeout={} -c lock_timeout={} -c idle_in_transaction_session_timeout=8000".format( + STATEMENT_TIMEOUT_MS, + LOCK_TIMEOUT_MS, + ), + }, + open=True, + ) + pool = redis.ConnectionPool.from_url( + self.redis_url(), + max_connections=REDIS_MAX_CONNECTIONS, + socket_timeout=REDIS_SOCKET_TIMEOUT, + socket_connect_timeout=REDIS_SOCKET_TIMEOUT, + health_check_interval=30, + decode_responses=True, + ) + self.redis = redis.Redis(connection_pool=pool) + self._wait_redis() + LOGGER.info("ready ttl=%s pool_max=%s", self.ttl, POOL_MAX) + + def shutdown(self): + self.shutting_down = True + if self.pool is not None: + self.pool.close() + if self.redis is not None: + self.redis.close() + + def ready(self): + if self.shutting_down or self.pool is None or self.redis is None: + return False + try: + with self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT 1") + self.redis.ping() + except Exception: + LOGGER.warning("ready check failed") + return False + return True + + def _wait_redis(self): + last = None + for _ in range(60): + try: + self.redis.ping() + return + except redis.RedisError as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("redis not ready: %s", last) + raise SystemExit(1) + + def _migrate(self, conninfo): + directory = Path(__file__).resolve().parents[1] / "migrations" + last = None + conn = None + for _ in range(60): + try: + conn = psycopg.connect(conninfo, connect_timeout=POOL_TIMEOUT_SECONDS, row_factory=dict_row) + conn.execute("SELECT 1") + conn.commit() + break + except psycopg.OperationalError as exc: + last = type(exc).__name__ + time.sleep(1) + else: + LOGGER.error("postgres not ready: %s", last) + raise SystemExit(1) + try: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK,)) + conn.commit() + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + for path in sorted(directory.glob("*.sql")): + version = path.stem + found = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (version,), + ).fetchone() + conn.rollback() + if found: + LOGGER.info("migration %s already applied", version) + continue + with conn.transaction(): + for statement in split_sql(path.read_text(encoding="utf-8")): + conn.execute(statement) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + LOGGER.info("migration %s applied", version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK,)) + conn.commit() + conn.close() + + def _connect(self): + try: + return self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) + except PoolTimeout as exc: + _db_down(exc) + + def create_user(self, username, display_name, password): + password_hash = hash_password(password) + user_id = uuid.uuid4() + created = utc_now() + try: + with self._connect() as conn: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (username) DO NOTHING + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash, created), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def login(self, username, password): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + verify_password(password, DUMMY_HASH) + return None + if not verify_password(password, row["password_hash"]): + return None + return self._issue(public_user(row)) + + def _issue(self, user): + token = secrets.token_urlsafe(32) + expires = utc_now() + timedelta(seconds=self.ttl) + payload = json.dumps( + {"user_id": user["id"], "expires_at": dump_time(expires)}, + separators=(",", ":"), + ) + # pxat — абсолютное время. Рестарт Redis не должен сдвигать срок. + # В значении тоже лежит expires_at: даже если TTL ключа разъедется, сессия кончится вовремя. + try: + self.redis.set(session_key(token), payload, pxat=int(expires.timestamp() * 1000)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return { + "token": token, + "token_type": "Bearer", + "expires_at": dump_time(expires), + "user": user, + } + + def user_for_token(self, token): + key = session_key(token) + try: + raw = self.redis.get(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + if not raw: + return None + try: + payload = json.loads(raw) + expires = parse_time(payload["expires_at"]) + user_id = payload["user_id"] + uuid.UUID(user_id) + except (ValueError, KeyError, TypeError): + return None + if utc_now() >= expires: + try: + self.redis.delete(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + return None + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (user_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def revoke(self, token): + try: + deleted = self.redis.delete(session_key(token)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return deleted == 1 + + def create_chat(self, title, member_ids, caller_id): + if caller_id not in member_ids: + raise ValueError("members") + chat_id = uuid.uuid4() + created = utc_now() + member_uuids = [uuid.UUID(item) for item in member_ids] + try: + with self._connect() as conn: + try: + with conn.transaction(): + # Сначала чат, потом участники. Ошибка внешнего ключа откатывает оба шага. + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created), + ) + for ordinal, member_id in enumerate(member_uuids): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, ordinal) + VALUES (%s, %s, %s) + """, + (chat_id, member_id, ordinal), + ) + row = conn.execute( + "SELECT id, title, created_at FROM chats WHERE id = %s", + (chat_id,), + ).fetchone() + except ForeignKeyViolation as exc: + raise ValueError("members") from exc + except psycopg.OperationalError as exc: + _db_down(exc) + return public_chat(row, member_ids) + + def _member(self, conn, chat_id, user_id): + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + + def get_chat(self, chat_id, user_id): + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s + GROUP BY c.id + """, + (chat_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_chat(row) + + def list_chats(self, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, "chats:" + user_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + sql = """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members m ON m.chat_id = c.id + """ + params = [user_id] + if after is not None: + sql += " WHERE (c.created_at, c.id) > (%s, %s)" + params.extend(after) + sql += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT %s" + params.append(limit + 1) + try: + with self._connect() as conn: + rows = conn.execute(sql, params).fetchall() + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_chat(row) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor("chats:" + user_id, last["created_at"], last["id"]) + return page, None + + def add_message(self, chat_id, sender_id, text): + message_id = uuid.uuid4() + created = utc_now() + try: + with self._connect() as conn: + with conn.transaction(): + if self._member(conn, chat_id, sender_id) is None: + return None + row = conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, body, created_at) + VALUES (%s, %s, %s, %s, %s) + RETURNING id, chat_id, sender_id, body AS text, created_at + """, + (message_id, chat_id, sender_id, text, created), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + return public_message(row) + + def list_messages(self, chat_id, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, chat_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + if after is None: + rows = conn.execute(SQL_MESSAGES_FIRST, (chat_id, limit + 1)).fetchall() + else: + rows = conn.execute( + SQL_MESSAGES_AFTER, + (chat_id, after[0], after[1], limit + 1), + ).fetchall() + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_message(row) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor(chat_id, last["created_at"], last["id"]) + return page, None diff --git a/student-tests/httpapi.py b/student-tests/httpapi.py new file mode 100644 index 00000000..85ef13aa --- /dev/null +++ b/student-tests/httpapi.py @@ -0,0 +1,36 @@ +import json +import urllib.error +import urllib.request + +BASE = "http://127.0.0.1:8080" + + +def call(method, path, body=None, headers=None, timeout=30): + hdrs = {"Accept": "application/json"} + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + hdrs["Content-Type"] = "application/json" + if headers: + hdrs.update(headers) + req = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method) + try: + response = urllib.request.urlopen(req, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + raw = response.read() + payload = None + if raw: + payload = json.loads(raw.decode("utf-8")) + return response.status, response.headers, payload + + +def bearer(token): + return {"Authorization": "Bearer " + token} + + +def basic(username, password): + import base64 + + raw = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii") + return {"Authorization": "Basic " + raw} diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 00000000..8e5c2cf3 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Проверки контракта и уровня 4, которым не нужно останавливать Redis/PostgreSQL.""" +import json +import time +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +from httpapi import basic, bearer, call +from sql import compose, psql + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" / "student_checks.txt" +LINES = [] + + +def log(text): + print(text) + LINES.append(text) + + +def check(cond, message): + if not cond: + log("FAIL: " + message) + raise SystemExit(1) + + +def register(username, display_name, password): + status, _, payload = call( + "POST", + "/api/v1/users", + {"username": username, "display_name": display_name, "password": password}, + ) + check(status == 201, "register {} -> {}".format(username, status)) + check("password" not in payload and "password_hash" not in payload, "password leaked") + return payload + + +def login(username, password): + status, headers, payload = call("POST", "/api/v1/auth/sessions", headers=basic(username, password)) + check(status == 201, "login -> {}".format(status)) + check(headers.get("X-Request-Id"), "no request id") + check(payload["token_type"] == "Bearer", "token type") + check(len(payload["token"]) >= 43, "token entropy/length") + check(username not in payload["token"], "token contains username") + return payload + + +def main(): + suffix = uuid.uuid4().hex[:8] + req = urllib.request.Request( + "http://127.0.0.1:8080/api/v1/users", + data=b"{not-json", + headers={"Content-Type": "application/json", "Accept": "application/json"}, + method="POST", + ) + try: + urllib.request.urlopen(req, timeout=30) + check(False, "bad json accepted") + except urllib.error.HTTPError as error: + body = json.loads(error.read().decode("utf-8")) + check(error.code == 400 and body["error"]["code"] == "invalid_request", "bad json") + check(body["error"]["request_id"] == error.headers["X-Request-Id"], "request id") + + status, _, _ = call( + "POST", + "/api/v1/users", + {"username": "abc", "display_name": "Имя", "password": "short-pass", "extra": 1}, + ) + check(status == 400, "extra field {}".format(status)) + status, _, _ = call( + "POST", + "/api/v1/users", + {"username": "ab", "display_name": "Имя", "password": "long-enough-pw"}, + ) + check(status == 400, "short username") + status, _, _ = call( + "POST", + "/api/v1/users", + {"username": "ok_" + suffix[:4], "display_name": "Имя", "password": "short"}, + ) + check(status == 400, "short password") + + password = "pw-" + suffix + "-ok" + alice = register("alice_" + suffix, "Алиса", password) + again, _, _ = call( + "POST", + "/api/v1/users", + {"username": alice["username"], "display_name": "Другая", "password": password + "-x"}, + ) + check(again == 409, "duplicate username {}".format(again)) + + session = login(alice["username"], password) + token = session["token"] + check(session["user"] == alice, "session user differs") + + status, headers, payload = call( + "POST", + "/api/v1/auth/sessions", + headers=basic(alice["username"], "wrong-password-1"), + ) + check(status == 401 and payload["error"]["code"] == "unauthorized", "wrong password") + check(headers.get("WWW-Authenticate", "").startswith("Basic"), "basic challenge") + status, _, payload_unknown = call( + "POST", + "/api/v1/auth/sessions", + headers=basic("missing_" + suffix, "wrong-password-1"), + ) + check(status == 401 and payload_unknown["error"]["code"] == payload["error"]["code"], "user enum") + + status, headers, _ = call("GET", "/api/v1/users/me", headers=basic(alice["username"], password)) + check(status == 401 and headers.get("WWW-Authenticate", "").startswith("Bearer"), "basic on business") + status, _, _ = call("GET", "/api/v1/users/me", headers={"X-User-Id": alice["id"]}) + check(status == 401, "x-user-id alone") + status, _, me = call( + "GET", + "/api/v1/users/me", + headers={**bearer(token), "X-User-Id": str(uuid.uuid4())}, + ) + check(status == 200 and me["id"] == alice["id"], "x-user-id overrode bearer") + + colon_pw = "secret:part-" + suffix + colon_user = register("colon_" + suffix, "Двоеточие", colon_pw) + colon_session = login(colon_user["username"], colon_pw) + check(colon_session["user"]["id"] == colon_user["id"], "colon password") + + uni_pw = "пароль-тест-" + suffix + uni_user = register("uni_" + suffix, "Юникод", uni_pw) + login(uni_user["username"], uni_pw) + + bob = register("bob_" + suffix, "Боб", password) + eve = register("eve_" + suffix, "Ева", password) + bob_token = login(bob["username"], password)["token"] + eve_token = login(eve["username"], password)["token"] + status, _, chat = call( + "POST", + "/api/v1/chats", + {"title": "Проверка", "member_ids": [alice["id"], bob["id"]]}, + bearer(token), + ) + check(status == 201, "chat {}".format(status)) + check(set(chat["member_ids"]) == {alice["id"], bob["id"]}, "members") + + status, _, _ = call( + "POST", + "/api/v1/chats", + {"title": "Без себя", "member_ids": [bob["id"]]}, + bearer(token), + ) + check(status == 400, "caller not in chat") + missing = str(uuid.uuid4()) + before = int(psql("SELECT count(*) FROM chats;")) + status, _, _ = call( + "POST", + "/api/v1/chats", + {"title": "Битый участник", "member_ids": [alice["id"], missing]}, + bearer(token), + ) + after = int(psql("SELECT count(*) FROM chats;")) + check(status == 400 and before == after, "chat rollback left a row {} {}".format(before, after)) + + psql("DELETE FROM chats WHERE title = 'rollback-probe';") + broken = compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-v", + "ON_ERROR_STOP=1", + "-q", + input_bytes=b""" + BEGIN; + INSERT INTO chats (id, title, created_at) + VALUES ('aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1', 'rollback-probe', now()); + INSERT INTO chat_members (chat_id, user_id, ordinal) + VALUES ('aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1', 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa2', 0); + COMMIT; + """, + check=False, + ) + check(broken.returncode != 0, "fk insert unexpectedly committed") + left = int(psql("SELECT count(*) FROM chats WHERE title = 'rollback-probe';")) + check(left == 0, "sql rollback probe left a chat") + log("rollback: api count {} -> {}, sql probe rows {}".format(before, after, left)) + + status, _, _ = call("GET", "/api/v1/chats/" + chat["id"], headers=bearer(eve_token)) + check(status == 404, "foreign chat") + status, _, _ = call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + {"text": "чужое"}, + bearer(eve_token), + ) + check(status == 404, "foreign message") + status, _, listed = call("GET", "/api/v1/chats?limit=100", headers=bearer(eve_token)) + check(status == 200 and all(item["id"] != chat["id"] for item in listed["items"]), "chat leak") + + sent = [] + for index in range(5): + actor = token if index % 2 == 0 else bob_token + status, _, message = call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + {"text": "сообщение {}".format(index)}, + bearer(actor), + ) + check(status == 201, "message {}".format(status)) + sent.append(message["id"]) + status, _, _ = call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + {"text": " "}, + bearer(token), + ) + check(status == 400, "blank message") + collected = [] + cursor = None + pages = 0 + while True: + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=2" + if cursor: + path += "&cursor=" + quote(cursor, safe="") + status, _, page = call("GET", path, headers=bearer(bob_token)) + check(status == 200, "page") + collected.extend(item["id"] for item in page["items"]) + pages += 1 + if page["next_cursor"] is None: + break + cursor = page["next_cursor"] + check(pages < 10, "cursor loop") + check(pages >= 3 and collected == sent, "pagination {} {}".format(pages, collected)) + other_status, _, other = call( + "POST", + "/api/v1/chats", + {"title": "Второй", "member_ids": [alice["id"], bob["id"]]}, + bearer(token), + ) + check(other_status == 201, "second chat") + call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + {"text": "для курсора"}, + bearer(token), + ) + call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + {"text": "ещё"}, + bearer(bob_token), + ) + _, _, first_page = call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=1", + headers=bearer(token), + ) + stolen, _, _ = call( + "GET", + "/api/v1/chats/" + other["id"] + "/messages?limit=1&cursor=" + quote(first_page["next_cursor"], safe=""), + headers=bearer(token), + ) + check(stolen == 400, "foreign cursor {}".format(stolen)) + + race_name = "race_" + suffix + race_password = "race-password-" + suffix + + def once(_n): + return call( + "POST", + "/api/v1/users", + {"username": race_name, "display_name": "Гонка", "password": race_password}, + )[0] + + with ThreadPoolExecutor(max_workers=12) as pool: + codes = list(pool.map(once, range(12))) + winners = codes.count(201) + conflicts = codes.count(409) + rows = int(psql("SELECT count(*) FROM users WHERE username = '{}';".format(race_name))) + check(winners == 1 and conflicts == 11 and rows == 1, "race {} rows {}".format(codes, rows)) + log("username race: one 201, eleven 409, one row") + + status, _, _ = call("DELETE", "/api/v1/auth/sessions/current", headers=bearer(token)) + check(status == 204, "logout {}".format(status)) + status, _, _ = call("GET", "/api/v1/users/me", headers=bearer(token)) + check(status == 401, "revoked token") + status, _, _ = call("DELETE", "/api/v1/auth/sessions/current", headers=bearer(token)) + check(status == 401, "second logout") + + users_before = psql("SELECT count(*) FROM users;") + versions_before = psql("SELECT version FROM schema_migrations ORDER BY version;") + compose("restart", "api") + deadline = time.time() + 60 + while True: + try: + ready, _, _ = call("GET", "/health/ready", timeout=3) + except Exception: + ready = 0 + if ready == 200: + break + if time.time() > deadline: + raise SystemExit("FAIL: api did not come back after migration restart") + time.sleep(0.5) + users_after = psql("SELECT count(*) FROM users;") + versions_after = psql("SELECT version FROM schema_migrations ORDER BY version;") + check(users_before == users_after and versions_before == versions_after, "migration duplicated data") + log("migrations reapplied, users {} -> {}, versions {}".format(users_before, users_after, versions_after)) + + redis_info = compose("exec", "-T", "redis", "redis-cli", "INFO", "persistence").stdout.decode("utf-8", "replace") + interesting = [ + line.strip() + for line in redis_info.splitlines() + if line.startswith(("aof_enabled", "aof_last_write_status", "aof_current_size", "rdb_last_bgsave_status")) + ] + log("redis persistence: " + ", ".join(interesting)) + check(any(line == "aof_enabled:1" for line in interesting), "aof is off") + log("PASS student checks") + EVIDENCE.write_text("\n".join(LINES) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + try: + main() + except SystemExit: + if LINES: + EVIDENCE.write_text("\n".join(LINES) + "\n", encoding="utf-8") + raise + except Exception as exc: + log("FAIL: {}: {}".format(type(exc).__name__, exc)) + EVIDENCE.write_text("\n".join(LINES) + "\n", encoding="utf-8") + raise SystemExit(1) diff --git a/student-tests/run_failures.py b/student-tests/run_failures.py new file mode 100644 index 00000000..0214babc --- /dev/null +++ b/student-tests/run_failures.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Отказ Redis/PostgreSQL и короткий TTL. В конце стенд возвращается к SESSION_TTL_SECONDS=3600.""" +import os +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path + +from httpapi import basic, bearer, call +from sql import ROOT, compose + +EVIDENCE = ROOT / "evidence" / "failures.txt" +LINES = [] + + +def log(text): + print(text) + LINES.append(text) + + +def check(cond, message): + if not cond: + log("FAIL: " + message) + raise SystemExit(1) + + +def wait_ready(timeout=60): + deadline = time.time() + timeout + while time.time() < deadline: + try: + status, _, _ = call("GET", "/health/ready", timeout=8) + except Exception: + status = 0 + if status == 200: + return + time.sleep(0.5) + raise SystemExit("FAIL: ready timeout") + + +def limited(payload): + text = "" if payload is None else str(payload).lower() + for word in ("redis", "postgres", "traceback", "select ", "password", "argon"): + check(word not in text, "error leaked " + word) + + +def recreate_api(ttl): + env = os.environ.copy() + env["SESSION_TTL_SECONDS"] = str(ttl) + proc = __import__("subprocess").run( + ["docker", "compose", "--env-file", ".env", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api"], + cwd=ROOT, + env=env, + stdout=__import__("subprocess").PIPE, + stderr=__import__("subprocess").PIPE, + timeout=180, + check=False, + ) + if proc.returncode != 0: + err = proc.stderr.decode("utf-8", "replace")[-500:] + raise SystemExit("FAIL: recreate api\n" + err) + wait_ready() + + +def main(): + suffix = uuid.uuid4().hex[:8] + password = "fail-password-" + suffix + username = "fail_" + suffix + status, _, user = call( + "POST", + "/api/v1/users", + {"username": username, "display_name": "Отказ", "password": password}, + ) + check(status == 201, "register") + status, _, session = call("POST", "/api/v1/auth/sessions", headers=basic(username, password)) + check(status == 201, "login") + token = session["token"] + + log("stop redis") + compose("stop", "redis") + try: + live, _, _ = call("GET", "/health/live", timeout=8) + ready, _, ready_body = call("GET", "/health/ready", timeout=15) + anon, anon_headers, anon_body = call("GET", "/api/v1/users/me", timeout=15) + xid, _, _ = call("GET", "/api/v1/users/me", headers={"X-User-Id": user["id"]}, timeout=15) + authed, _, authed_body = call("GET", "/api/v1/users/me", headers=bearer(token), timeout=15) + check(live == 200, "live during redis stop {}".format(live)) + check(ready == 503, "ready during redis stop {}".format(ready)) + check(anon == 401 and anon_headers.get("WWW-Authenticate", "").startswith("Bearer"), "anon {}".format(anon)) + check(xid == 401, "x-user-id during redis stop {}".format(xid)) + check(authed == 503, "bearer during redis stop {}".format(authed)) + limited(ready_body) + limited(anon_body) + limited(authed_body) + log("redis down: live 200, ready 503, anon/x-user-id 401, bearer 503") + finally: + compose("start", "redis") + wait_ready() + + log("stop postgres") + compose("stop", "postgres") + try: + live, _, _ = call("GET", "/health/live", timeout=8) + ready, _, _ = call("GET", "/health/ready", timeout=15) + anon, _, _ = call("GET", "/api/v1/users/me", timeout=15) + authed, _, authed_body = call("GET", "/api/v1/users/me", headers=bearer(token), timeout=15) + check(live == 200, "live during postgres stop {}".format(live)) + check(ready == 503, "ready during postgres stop {}".format(ready)) + check(anon == 401, "anon during postgres stop {}".format(anon)) + check(authed == 503, "bearer during postgres stop {}".format(authed)) + limited(authed_body) + log("postgres down: live 200, ready 503, anon 401, bearer 503") + finally: + compose("start", "postgres") + wait_ready() + + log("session ttl 2s") + recreate_api(2) + try: + status, _, short = call("POST", "/api/v1/auth/sessions", headers=basic(username, password)) + check(status == 201, "short login") + expires = datetime.fromisoformat(short["expires_at"].replace("Z", "+00:00")) + delta = (expires - datetime.now(timezone.utc)).total_seconds() + check(1 <= delta <= 4, "ttl delta {}".format(delta)) + time.sleep(delta + 1.2) + status, headers, _ = call("GET", "/api/v1/users/me", headers=bearer(short["token"]), timeout=15) + check(status == 401 and headers.get("WWW-Authenticate", "").startswith("Bearer"), "expired {}".format(status)) + log("expired session -> 401, delta was {:.2f}s".format(delta)) + finally: + recreate_api(3600) + status, _, restored = call("POST", "/api/v1/auth/sessions", headers=basic(username, password)) + check(status == 201, "ttl restored login") + expires = datetime.fromisoformat(restored["expires_at"].replace("Z", "+00:00")) + delta = (expires - datetime.now(timezone.utc)).total_seconds() + check(delta > 300, "ttl not restored {}".format(delta)) + log("ttl restored, expires in {:.0f}s".format(delta)) + + log("PASS failure checks") + EVIDENCE.write_text("\n".join(LINES) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + try: + main() + except SystemExit: + EVIDENCE.write_text("\n".join(LINES) + "\n", encoding="utf-8") + raise + except Exception as exc: + log("FAIL: {}: {}".format(type(exc).__name__, exc)) + EVIDENCE.write_text("\n".join(LINES) + "\n", encoding="utf-8") + raise SystemExit(1) diff --git a/student-tests/run_track2.py b/student-tests/run_track2.py new file mode 100644 index 00000000..8d9a2f87 --- /dev/null +++ b/student-tests/run_track2.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Трек 2: план keyset-запроса на неравномерной истории и backup в новый volume.""" +import json +import re +import statistics +import subprocess +import time +import uuid +from pathlib import Path + +from httpapi import bearer, call +from sql import ROOT, compose, psql, psql_file + +EVIDENCE = ROOT / "evidence" +BACKUPS = ROOT / "backups" +RESTORE_NAME = "mtusi-lab2-restore" +RESTORE_VOLUME = "mtusi-lab2-restore-data" +INSERT_ROWS = 20000 +REPEATS = 3 + + +def log(text): + print(text, flush=True) + + +def check(cond, message): + if not cond: + raise SystemExit("FAIL: " + message) + + +def postgres_image(): + text = (ROOT / "compose.yaml").read_text(encoding="utf-8") + match = re.search(r"image:\s*(postgres:\S+)", text) + check(match is not None, "postgres image tag") + return match.group(1) + + +def median(values): + return statistics.median(values) + + +def execution_ms(plan): + found = re.findall(r"Execution Time:\s+([0-9.]+)\s+ms", plan) + check(found, "no execution time in plan") + return float(found[-1]) + + +def planning_ms(plan): + found = re.findall(r"Planning Time:\s+([0-9.]+)\s+ms", plan) + return float(found[-1]) if found else None + + +def first_buffers(plan): + found = re.findall(r"Buffers:.*", plan) + return found[0].strip() if found else "" + + +def run_insert(label): + sql = """ + EXPLAIN (ANALYZE, BUFFERS) + INSERT INTO track2_write (id, chat_id, sender_id, body, created_at) + SELECT gen_random_uuid(), + '00000000-0000-4000-8000-000000000001'::uuid, + '00000000-0000-4000-8000-000000000002'::uuid, + 'w', + timestamptz '2025-01-01 00:00:00+00' + (n || ' microseconds')::interval + FROM generate_series(1, {rows}) AS n; + """.format(rows=INSERT_ROWS) + samples = [] + psql("TRUNCATE track2_write;") + psql_file(sql) + for _ in range(REPEATS): + psql("TRUNCATE track2_write;") + samples.append(execution_ms(psql_file(sql))) + result = {"label": label, "rows": INSERT_ROWS, "repeats": samples, "median_ms": median(samples)} + log("{} median {:.1f} ms {}".format(label, result["median_ms"], ["{:.1f}".format(x) for x in samples])) + return result + + +def psql_on(container, sql): + proc = subprocess.run( + ["docker", "exec", "-i", container, "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-q", "-t", "-A"], + input=sql.encode("utf-8"), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError(proc.stderr.decode("utf-8", "replace")[-500:]) + return proc.stdout.decode("utf-8").strip() + + +def main(): + EVIDENCE.mkdir(exist_ok=True) + BACKUPS.mkdir(exist_ok=True) + source = (ROOT / "src" / "store.py").read_text(encoding="utf-8") + check("AND (created_at, id) > (%s, %s)" in source, "app query changed") + + password = "track2-" + uuid.uuid4().hex + username = "track2_sender" + psql( + """ + DELETE FROM messages + WHERE sender_id IN (SELECT id FROM users WHERE username = 'track2_sender') + OR chat_id IN (SELECT id FROM chats WHERE title LIKE 'track2:%'); + DELETE FROM chat_members + WHERE user_id IN (SELECT id FROM users WHERE username = 'track2_sender') + OR chat_id IN (SELECT id FROM chats WHERE title LIKE 'track2:%'); + DELETE FROM chats WHERE title LIKE 'track2:%'; + DELETE FROM users WHERE username = 'track2_sender'; + DROP TABLE IF EXISTS track2_write; + """ + ) + status, _, user = call( + "POST", + "/api/v1/users", + {"username": username, "display_name": "Трек 2", "password": password}, + ) + check(status == 201, "register track2 {}".format(status)) + status, _, session = call( + "POST", + "/api/v1/auth/sessions", + headers=__import__("httpapi").basic(username, password), + ) + check(status == 201, "login track2") + user_id = user["id"] + + specs = [("track2:big", 70000, 0), ("track2:mid", 20000, 1)] + for index in range(10): + specs.append(("track2:m{:02d}".format(index), 1000, 2 + index)) + for index in range(100): + specs.append(("track2:s{:02d}".format(index), 10, 12 + index)) + check(sum(item[1] for item in specs) == 101000, "distribution") + + chat_rows = [] + for title, _count, _idx in specs: + chat_rows.append( + "('{}', '{}', timestamptz '2024-06-01 00:00:00+00')".format(str(uuid.uuid4()), title) + ) + psql("INSERT INTO chats (id, title, created_at) VALUES\n" + ",\n".join(chat_rows) + ";") + psql( + """ + INSERT INTO chat_members (chat_id, user_id, ordinal) + SELECT id, '{}'::uuid, 0 FROM chats WHERE title LIKE 'track2:%'; + """.format(user_id) + ) + + statements = ["BEGIN;"] + for title, count, index in specs: + statements.append( + """ + INSERT INTO messages (id, chat_id, sender_id, body, created_at) + SELECT gen_random_uuid(), + c.id, + '{user}'::uuid, + 't2-' || n::text, + timestamptz '2024-06-01 00:00:00+00' + + ((n * 1000 + {index}) || ' microseconds')::interval + FROM chats c + CROSS JOIN generate_series(1, {count}) AS n + WHERE c.title = '{title}'; + """.format(user=user_id, index=index, count=count, title=title) + ) + statements.append("COMMIT;") + log("loading 101000 messages") + started = time.perf_counter() + psql("\n".join(statements), timeout=300) + load_seconds = time.perf_counter() - started + psql("ANALYZE messages; ANALYZE chats; ANALYZE chat_members;") + total = int(psql("SELECT count(*) FROM messages WHERE chat_id IN (SELECT id FROM chats WHERE title LIKE 'track2:%');")) + check(total == 101000, "loaded {}".format(total)) + log("loaded in {:.1f}s".format(load_seconds)) + + big_id = psql("SELECT id FROM chats WHERE title = 'track2:big';") + small_id = psql("SELECT id FROM chats WHERE title = 'track2:s00';") + cursor = psql( + """ + SELECT created_at::text || '|' || id::text + FROM messages + WHERE chat_id = '{chat}' AND body = 't2-5'; + """.format(chat=big_id) + ) + created_at, message_id = cursor.split("|", 1) + check(re.fullmatch(r"[0-9a-f-]{36}", message_id), "cursor id") + + def keyset_sql(): + return """ + EXPLAIN (ANALYZE, BUFFERS) + SELECT id, chat_id, sender_id, body AS text, created_at + FROM messages + WHERE chat_id = '{chat}'::uuid + AND (created_at, id) > (timestamptz '{ts}', '{mid}'::uuid) + ORDER BY created_at, id + LIMIT 51; + """.format(chat=big_id, ts=created_at, mid=message_id) + + psql_file(keyset_sql()) + good_plan = psql_file(keyset_sql()) + (EVIDENCE / "explain_history_index.txt").write_text(good_plan, encoding="utf-8") + check("messages_history_idx" in good_plan, "history index was not used") + + first_plan = psql_file( + """ + EXPLAIN (ANALYZE, BUFFERS) + SELECT id, chat_id, sender_id, body AS text, created_at + FROM messages + WHERE chat_id = '{chat}'::uuid + ORDER BY created_at, id + LIMIT 51; + """.format(chat=big_id) + ) + (EVIDENCE / "explain_history_first_page.txt").write_text(first_plan, encoding="utf-8") + + offset_plan = psql_file( + """ + EXPLAIN (ANALYZE, BUFFERS) + SELECT id, chat_id, sender_id, body AS text, created_at + FROM messages + WHERE chat_id = '{chat}'::uuid + ORDER BY created_at, id + OFFSET 5000 + LIMIT 50; + """.format(chat=big_id) + ) + (EVIDENCE / "explain_offset.txt").write_text(offset_plan, encoding="utf-8") + + weak_script = """ + BEGIN; + DROP INDEX messages_history_idx; + CREATE INDEX messages_created_idx ON messages (created_at); + ANALYZE messages; + {explain} + ROLLBACK; + """.format(explain=keyset_sql()) + weak_plan = psql_file(weak_script) + (EVIDENCE / "explain_created_at_index.txt").write_text(weak_plan, encoding="utf-8") + still = psql("SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history_idx';") + check("chat_id, created_at, id" in still, "history index missing after rollback") + check(psql("SELECT count(*) FROM pg_indexes WHERE indexname = 'messages_created_idx';") == "0", "weak index left behind") + + status, _, page = call( + "GET", + "/api/v1/chats/" + big_id + "/messages?limit=2", + headers=bearer(session["token"]), + ) + check(status == 200 and len(page["items"]) == 2, "api page") + sql_first = psql( + """ + SELECT body FROM messages + WHERE chat_id = '{chat}' + ORDER BY created_at, id + LIMIT 2; + """.format(chat=big_id) + ).splitlines() + check([item["text"] for item in page["items"]] == sql_first, "api order != sql") + + log("write cost") + psql( + """ + CREATE TABLE track2_write ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + body text NOT NULL, + created_at timestamptz NOT NULL + ); + """ + ) + try: + pk_only = run_insert("pk_only") + psql("CREATE INDEX track2_write_hist ON track2_write (chat_id, created_at, id);") + with_hist = run_insert("pk_and_history") + psql("CREATE INDEX track2_write_body ON track2_write (chat_id, body);") + with_body = run_insert("pk_history_and_body") + finally: + psql("DROP TABLE IF EXISTS track2_write;") + + log("backup") + dump_started = time.perf_counter() + dump = compose( + "exec", + "-T", + "postgres", + "pg_dump", + "-U", + "messenger", + "-d", + "messenger", + "-Fc", + "--no-owner", + "--no-acl", + timeout=300, + ) + dump_seconds = time.perf_counter() - dump_started + dump_path = BACKUPS / "track2.dump" + dump_path.write_bytes(dump.stdout) + log("dump {:.2f}s, {} bytes".format(dump_seconds, len(dump.stdout))) + + image = postgres_image() + db_password = (ROOT / "secrets" / "postgres_password").read_text(encoding="utf-8").strip() + subprocess.run(["docker", "rm", "-f", RESTORE_NAME], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + subprocess.run(["docker", "volume", "rm", RESTORE_VOLUME], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + subprocess.run(["docker", "volume", "create", RESTORE_VOLUME], check=True, stdout=subprocess.PIPE) + started_container = subprocess.run( + [ + "docker", "run", "-d", + "--name", RESTORE_NAME, + "--network", "mtusi-lab2-net", + "-e", "POSTGRES_USER=messenger", + "-e", "POSTGRES_PASSWORD=" + db_password, + "-e", "POSTGRES_DB=messenger", + "-v", RESTORE_VOLUME + ":/var/lib/postgresql/data", + image, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + check(started_container.returncode == 0, "restore container") + try: + deadline = time.time() + 60 + while time.time() < deadline: + probe = subprocess.run( + ["docker", "exec", RESTORE_NAME, "pg_isready", "-U", "messenger", "-d", "messenger"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if probe.returncode == 0: + break + time.sleep(0.5) + else: + raise SystemExit("FAIL: restore postgres not ready") + restore_started = time.perf_counter() + restored = subprocess.run( + [ + "docker", "exec", "-i", RESTORE_NAME, + "pg_restore", "-U", "messenger", "-d", "messenger", + "--no-owner", "--no-acl", "--exit-on-error", + ], + input=dump.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + restore_seconds = time.perf_counter() - restore_started + if restored.returncode != 0: + err = restored.stderr.decode("utf-8", "replace") + err = err.replace(db_password, "***") + raise SystemExit("FAIL: pg_restore\n" + err[-800:]) + log("restore {:.2f}s".format(restore_seconds)) + + def counts(sql): + return { + "messages": int(sql("SELECT count(*) FROM messages;")), + "users": int(sql("SELECT count(*) FROM users;")), + "chats": int(sql("SELECT count(*) FROM chats;")), + "members": int(sql("SELECT count(*) FROM chat_members;")), + "big": int(sql("SELECT count(*) FROM messages WHERE chat_id = '{}'::uuid;".format(big_id))), + "small": int(sql("SELECT count(*) FROM messages WHERE chat_id = '{}'::uuid;".format(small_id))), + "small_bodies": sql( + """ + SELECT string_agg(body, ',' ORDER BY created_at, id) + FROM messages WHERE chat_id = '{}'::uuid; + """.format(small_id) + ), + "markers": sql( + """ + SELECT string_agg(body, ',' ORDER BY created_at, id) + FROM messages + WHERE chat_id = '{}'::uuid AND body IN ('t2-1', 't2-500', 't2-70000'); + """.format(big_id) + ), + } + + src_counts = counts(psql) + dst_counts = counts(lambda sql: psql_on(RESTORE_NAME, sql)) + check(src_counts == dst_counts, "restore mismatch {} {}".format(src_counts, dst_counts)) + log("counts match {}".format({k: src_counts[k] for k in ("messages", "users", "chats", "members", "big", "small")})) + finally: + subprocess.run(["docker", "rm", "-f", RESTORE_NAME], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + subprocess.run(["docker", "volume", "rm", RESTORE_VOLUME], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) + if dump_path.exists(): + dump_path.unlink() + + settings = psql( + """ + SELECT name || '=' || setting + FROM pg_settings + WHERE name IN ('fsync', 'synchronous_commit', 'full_page_writes', 'wal_level') + ORDER BY name; + """ + ) + version = psql("SELECT version();") + size = psql("SELECT pg_size_pretty(pg_total_relation_size('messages'));") + info = subprocess.run( + ["docker", "info", "--format", "{{.NCPU}}|{{.MemTotal}}|{{.OperatingSystem}}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + text=True, + ) + summary = { + "loaded_messages": total, + "load_seconds": round(load_seconds, 2), + "distribution": { + "big": 70000, + "mid": 20000, + "medium_10x1000": 10000, + "small_100x10": 1000, + }, + "messages_relation_size": size, + "history_index_deep": { + "execution_ms": execution_ms(good_plan), + "planning_ms": planning_ms(good_plan), + "buffers": first_buffers(good_plan), + }, + "history_index_first_page": { + "execution_ms": execution_ms(first_plan), + "buffers": first_buffers(first_plan), + }, + "offset_5000": { + "execution_ms": execution_ms(offset_plan), + "buffers": first_buffers(offset_plan), + }, + "created_at_index_deep": { + "execution_ms": execution_ms(weak_plan), + "buffers": first_buffers(weak_plan), + }, + "write_cost": [pk_only, with_hist, with_body], + "dump_seconds": round(dump_seconds, 3), + "restore_seconds": round(restore_seconds, 3), + "counts": src_counts, + "postgres": version, + "durability": settings.splitlines(), + "docker_info": info.stdout.strip(), + "api_first_page_matches_sql": True, + } + (EVIDENCE / "track2_summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + log("PASS track2") + + +if __name__ == "__main__": + main() diff --git a/student-tests/sql.py b/student-tests/sql.py new file mode 100644 index 00000000..38a86117 --- /dev/null +++ b/student-tests/sql.py @@ -0,0 +1,61 @@ +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args, check=True, input_bytes=None, timeout=180): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + if check and proc.returncode != 0: + err = proc.stderr.decode("utf-8", "replace")[-800:] + raise RuntimeError(err or "docker compose failed") + return proc + + +def psql(sql, timeout=180): + proc = compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-v", + "ON_ERROR_STOP=1", + "-q", + "-t", + "-A", + input_bytes=sql.encode("utf-8"), + timeout=timeout, + ) + return proc.stdout.decode("utf-8").strip() + + +def psql_file(sql, timeout=180): + """Текст плана, без -t, чтобы EXPLAIN остался читаемым.""" + proc = compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-v", + "ON_ERROR_STOP=1", + "-X", + input_bytes=sql.encode("utf-8"), + timeout=timeout, + ) + return proc.stdout.decode("utf-8") -- 2.54.0 From 06fa02c8023f353ca5612446833448798d775534 Mon Sep 17 00:00:00 2001 From: Wladislava1 <146841829+Wladislava1@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:47:15 +0300 Subject: [PATCH 2/2] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20500=20=D0=BD=D0=B0=20=D0=B4=D1=83=D0=B1=D0=BB?= =?UTF-8?q?=D0=B8=D0=BA=D0=B0=D1=82=D0=B0=D1=85=20member=5Fids=20=D0=B2=20?= =?UTF-8?q?=D1=80=D0=B0=D0=B7=D0=BD=D0=BE=D0=BC=20=D1=80=D0=B5=D0=B3=D0=B8?= =?UTF-8?q?=D1=81=D1=82=D1=80=D0=B5=20UUID.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Нормализуем UUID до проверки уникальности и отвечаем 400 invalid_request вместо UniqueViolation. Co-authored-by: Cursor --- evidence/student_checks.txt | 6 +++--- src/schemas.py | 12 ++++++++---- src/store.py | 25 +++++++++++++++++++++---- student-tests/run.py | 14 ++++++++++++++ 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/evidence/student_checks.txt b/evidence/student_checks.txt index a0afa264..23719732 100644 --- a/evidence/student_checks.txt +++ b/evidence/student_checks.txt @@ -1,5 +1,5 @@ -rollback: api count 119 -> 119, sql probe rows 0 +rollback: api count 125 -> 125, sql probe rows 0 username race: one 201, eleven 409, one row -migrations reapplied, users 26 -> 26, versions 001_init -redis persistence: rdb_last_bgsave_status:ok, aof_enabled:1, aof_last_write_status:ok, aof_current_size:7677 +migrations reapplied, users 46 -> 46, versions 001_init +redis persistence: rdb_last_bgsave_status:ok, aof_enabled:1, aof_last_write_status:ok, aof_current_size:14166 PASS student checks diff --git a/src/schemas.py b/src/schemas.py index a8b41280..b1d9e4d8 100644 --- a/src/schemas.py +++ b/src/schemas.py @@ -58,18 +58,22 @@ class CreateChat(StrictModel): def member_ids_ok(cls, value): if not isinstance(value, list) or not (1 <= len(value) <= 100): raise ValueError("member_ids") + # Сравниваем и возвращаем канонический UUID (нижний регистр + дефисы), + # иначе u и u.upper() проходят как «разные» строки и падают UniqueViolation в БД. + normalized = [] seen = set() for item in value: if not isinstance(item, str): raise ValueError("member_ids") try: - uuid.UUID(item) + canonical = str(uuid.UUID(item)) except ValueError as exc: raise ValueError("member_ids") from exc - if item in seen: + if canonical in seen: raise ValueError("member_ids") - seen.add(item) - return value + seen.add(canonical) + normalized.append(canonical) + return normalized class CreateMessage(StrictModel): diff --git a/src/store.py b/src/store.py index 5e6c8928..20278f62 100644 --- a/src/store.py +++ b/src/store.py @@ -11,7 +11,7 @@ from pathlib import Path import psycopg import redis -from psycopg.errors import ForeignKeyViolation +from psycopg.errors import ForeignKeyViolation, UniqueViolation from psycopg.rows import dict_row from psycopg_pool import ConnectionPool, PoolTimeout @@ -414,11 +414,25 @@ class Store: return deleted == 1 def create_chat(self, title, member_ids, caller_id): - if caller_id not in member_ids: + # Ещё раз через UUID: дубликаты в разном регистре уже отсекаются в schemas, + # здесь — страховка и единый канонический вид для caller/members. + try: + caller_uuid = uuid.UUID(str(caller_id)) + member_uuids = [] + seen = set() + for item in member_ids: + member_uuid = uuid.UUID(str(item)) + if member_uuid in seen: + raise ValueError("members") + seen.add(member_uuid) + member_uuids.append(member_uuid) + except ValueError as exc: + raise ValueError("members") from exc + if caller_uuid not in seen: raise ValueError("members") chat_id = uuid.uuid4() created = utc_now() - member_uuids = [uuid.UUID(item) for item in member_ids] + canonical_members = [str(item) for item in member_uuids] try: with self._connect() as conn: try: @@ -442,9 +456,12 @@ class Store: ).fetchone() except ForeignKeyViolation as exc: raise ValueError("members") from exc + except UniqueViolation as exc: + # Дубликат участника, если обошли schema-валидацию. + raise ValueError("members") from exc except psycopg.OperationalError as exc: _db_down(exc) - return public_chat(row, member_ids) + return public_chat(row, canonical_members) def _member(self, conn, chat_id, user_id): return conn.execute( diff --git a/student-tests/run.py b/student-tests/run.py index 8e5c2cf3..8d6490fd 100644 --- a/student-tests/run.py +++ b/student-tests/run.py @@ -162,6 +162,20 @@ def main(): after = int(psql("SELECT count(*) FROM chats;")) check(status == 400 and before == after, "chat rollback left a row {} {}".format(before, after)) + # Один UUID дважды в разном регистре — 400, не 500 UniqueViolation. + before_case = int(psql("SELECT count(*) FROM chats;")) + status, _, body_case = call( + "POST", + "/api/v1/chats", + {"title": "Дубликат UUID", "member_ids": [alice["id"], alice["id"].upper()]}, + bearer(token), + ) + after_case = int(psql("SELECT count(*) FROM chats;")) + check( + status == 400 and before_case == after_case and body_case.get("error", {}).get("code") == "invalid_request", + "case-duplicate member_ids {} chats {}->{}".format(status, before_case, after_case), + ) + psql("DELETE FROM chats WHERE title = 'rollback-probe';") broken = compose( "exec", -- 2.54.0