From 41e8d67a1a57fcc3e32d62e06b0fa759a6bcc6ee Mon Sep 17 00:00:00 2001 From: Fanaenic Date: Thu, 8 Oct 2026 14:14:20 +0300 Subject: [PATCH] =?UTF-8?q?Lab2-=20=D0=BE=D1=86=D0=B5=D0=BD=D0=BA=D0=B0=20?= =?UTF-8?q?5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 +- .github/workflows/app.yml | 23 ++ Dockerfile | 22 + Makefile | 17 +- REPORT.md | 179 ++++++++- compose.yaml | 142 +++++++ evidence/check.txt | 3 + evidence/docker.txt | 12 + evidence/durability.txt | 6 + evidence/load-32.json | 17 + evidence/load.json | 17 + evidence/migrations.txt | 5 + evidence/persistence.txt | 16 + evidence/quality.txt | 6 + evidence/smoke.txt | 3 + evidence/student-tests.json | 74 ++++ evidence/student-tests.txt | 16 + .../track2/explain_pk_only_deep_keyset.txt | 1 + .../track2/explain_production_deep_keyset.txt | 9 + evidence/track2/results.json | 361 +++++++++++++++++ evidence/validate.txt | 3 + infra/valkey/valkey.conf | 8 + scripts/init_secrets.py | 39 ++ scripts/restart.cmd | 5 + scripts/restart.sh | 6 + scripts/track2/run_track2.py | 284 +++++++++++++ scripts/track2/seed.sql | 35 ++ src/cmd/server/main.go | 186 +++++++++ src/go.mod | 21 + src/go.sum | 42 ++ src/internal/api/handlers.go | 377 ++++++++++++++++++ src/internal/api/helpers.go | 94 +++++ src/internal/api/middleware.go | 138 +++++++ src/internal/api/router.go | 45 +++ src/internal/apperr/errors.go | 64 +++ src/internal/auth/password.go | 100 +++++ src/internal/auth/sessions.go | 137 +++++++ src/internal/cache/valkey.go | 80 ++++ src/internal/db/migrate.go | 122 ++++++ src/internal/db/migrations/0001_init.sql | 51 +++ src/internal/db/pool.go | 31 ++ src/internal/model/models.go | 51 +++ src/internal/store/cursor.go | 33 ++ src/internal/store/postgres.go | 306 ++++++++++++++ student-tests/run_tests.py | 294 ++++++++++++++ 45 files changed, 3467 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/app.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/check.txt create mode 100644 evidence/docker.txt create mode 100644 evidence/durability.txt create mode 100644 evidence/load-32.json create mode 100644 evidence/load.json create mode 100644 evidence/migrations.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/quality.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/student-tests.json create mode 100644 evidence/student-tests.txt create mode 100644 evidence/track2/explain_pk_only_deep_keyset.txt create mode 100644 evidence/track2/explain_production_deep_keyset.txt create mode 100644 evidence/track2/results.json create mode 100644 evidence/validate.txt create mode 100644 infra/valkey/valkey.conf create mode 100644 scripts/init_secrets.py create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 scripts/track2/run_track2.py create mode 100644 scripts/track2/seed.sql create mode 100644 src/cmd/server/main.go create mode 100644 src/go.mod create mode 100644 src/go.sum create mode 100644 src/internal/api/handlers.go create mode 100644 src/internal/api/helpers.go create mode 100644 src/internal/api/middleware.go create mode 100644 src/internal/api/router.go create mode 100644 src/internal/apperr/errors.go create mode 100644 src/internal/auth/password.go create mode 100644 src/internal/auth/sessions.go create mode 100644 src/internal/cache/valkey.go create mode 100644 src/internal/db/migrate.go create mode 100644 src/internal/db/migrations/0001_init.sql create mode 100644 src/internal/db/pool.go create mode 100644 src/internal/model/models.go create mode 100644 src/internal/store/cursor.go create mode 100644 src/internal/store/postgres.go create mode 100644 student-tests/run_tests.py diff --git a/.env.example b/.env.example index 714c8076..3bfc3c4b 100644 --- a/.env.example +++ b/.env.example @@ -7,5 +7,7 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url +SESSION_CACHE_MAX_TTL_SECONDS=60 +DB_MAX_CONNS=10 diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..b13e86d4 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,23 @@ +name: Application (lab2) +on: [push, pull_request] +permissions: + contents: read +jobs: + app: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python scripts/init_secrets.py + - run: docker compose config --quiet + - run: docker compose up -d --build --wait api + - run: make test PYTHON=python + - run: make test-persistence PYTHON=python ACTION_SCRIPT=scripts/restart.sh + - run: python student-tests/run_tests.py + - if: failure() + run: docker compose logs --no-color --tail 200 + - if: always() + run: docker compose --profile ttl down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..47fee6af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.23.4-alpine3.21 AS build +WORKDIR /src + +COPY src/go.mod src/go.sum ./ +RUN go mod download + +COPY src/ ./ +ENV CGO_ENABLED=0 +RUN go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server + +FROM alpine:3.21.2 AS runtime +RUN addgroup -S app && adduser -S -G app app +WORKDIR /app +COPY --from=build /out/server /app/server +USER app +EXPOSE 8080 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1 +ENTRYPOINT ["/app/server"] diff --git a/Makefile b/Makefile index 9ecb223e..a3d1f037 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence secrets up down test-student track2 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,3 +24,18 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +secrets: + $(PYTHON) scripts/init_secrets.py + +up: secrets + docker compose up -d --build --wait api + +down: + docker compose --profile ttl --profile research stop + +test-student: + $(PYTHON) student-tests/run_tests.py --base-url "$(BASE_URL)" --json evidence/student-tests.json + +track2: + $(PYTHON) scripts/track2/run_track2.py --repeat 21 diff --git a/REPORT.md b/REPORT.md index e42f2b05..a9c13eba 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,184 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кораблев Петр, БВТ2304 +- Целевая оценка: 5 +- Выбранный трек 5: трек 2 — план запроса и восстановление +- Commit/PR: solution/BVT2304-korablev +- Использованные библиотеки: pgx/v5 (pgxpool) — PostgreSQL, go-redis/v9 — Valkey, golang.org/x/crypto/argon2 — хеширование паролей, google/uuid; логирование — стандартный `log/slog`. +- Помощь ИИ использовалась для создания отчета. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker 26+ с Compose v2, Python 3.9+ (тесты и генерация секретов), make — по желанию. Go локально не нужен: приложение собирается в Docker (`golang:1.23.4-alpine3.21` → `alpine:3.21.2`, non-root). Версии образов зафиксированы: `postgres:17.2-alpine3.21`, `valkey/valkey:8.0.2-alpine3.21`. + +```sh +make secrets # python scripts/init_secrets.py — secrets/* со случайными паролями и .env из .env.example +make up # docker compose up -d --build --wait api +``` + +Порядок старта: postgres → `migrate` (одноразовый bootstrap: версионируемые миграции, advisory lock, повторный запуск безопасен) → valkey → api. API слушает только `127.0.0.1:8080`; порты PostgreSQL и Valkey наружу не публикуются. + +Проверки: + +```sh +make check +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-student # собственные тесты уровня 4; останавливают и запускают postgres/valkey +make track2 # эксперимент трека 2 (--repeat 21) в отдельных контейнерах профиля research +``` + +Без make — те же команды через `python`: `scripts/check_contract.py`, `tests/smoke.py`, `tests/resilience.py --mode persistence --action-script scripts/restart.sh`, `student-tests/run_tests.py`, `scripts/track2/run_track2.py`. + +На Windows Python не умеет запускать `.sh` через `subprocess`, поэтому рядом лежит `scripts/restart.cmd` с теми же командами Compose; на машине автора тест сохранности запускался с ним, а сам `restart.sh` проверен из Git Bash (rc=0). + +Безопасная остановка: `make down` (`docker compose stop`) — данные остаются в volume `mtusi-lab2_pgdata`. Намеренное удаление всех учебных данных: + +```sh +docker compose --profile ttl --profile research down -v +``` + +Секреты (`secrets/`, `.env`) создаются локально, исключены из Git и из Docker build context и передаются контейнерам как Compose secrets (`/run/secrets/...`). Приложение читает `DATABASE_URL_FILE` / `CACHE_URL_FILE` и не логирует их значения. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HTTP (`net/http`) → обработчики → Store на PostgreSQL (pgxpool, не более 10 соединений). Сессии: PostgreSQL как источник истины, Valkey как кэш перед ним. Из lab1 перенесены HTTP-слой, формат ошибок, request-id, валидация, keyset-курсоры `(created_at, id)` и graceful shutdown. In-memory Store заменён на PostgreSQL с теми же операциями, `X-User-Id` — на Bearer-сессии. + +Схема (`src/internal/db/migrations/0001_init.sql`): `users` (UNIQUE username, CHECK формата), `chats`, `chat_members` (PK `(chat_id, user_id)`, `position` хранит порядок `member_ids`), `messages`, `sessions`. В `messages` стоит составной FK `(chat_id, sender_id) → chat_members`: база сама гарантирует, что автор сообщения — участник чата. История читается по индексу `(chat_id, created_at, id)`. Время назначает PostgreSQL с точностью до миллисекунд, так что все реплики используют одни часы. + +Создание чата атомарно: `INSERT chats` и `INSERT chat_members` идут в одной транзакции. Неизвестный участник нарушает FK, транзакция откатывается целиком, клиент получает 400. Гонку регистрации одинакового username решает `UNIQUE` (23505 → 409), а не связка «SELECT, потом INSERT». Все SQL параметризованы (`$1…`). У пула есть `ConnectTimeout` 3 с; на сервере заданы `statement_timeout` 5 с, `lock_timeout` 2 с и `idle_in_transaction_session_timeout` 10 с. Каждый HTTP-запрос ограничен контекстом в 8 с. + +Пароли хешируются Argon2id (m=19 MiB, t=2, p=1, соль 16 байт). Одновременно считается не больше 4 хешей, чтобы ограничить память. Для неизвестного username считается dummy-hash, поэтому ответ — тот же 401 за сопоставимое время. Токен состоит из 32 случайных байт (256 бит) в base64url. В БД и в ключах Valkey хранится только его SHA-256. Быстрый хеш здесь уместен: перебирать 256 бит бессмысленно, а у пароля энтропия низкая, поэтому для него нужен медленный KDF. `expires_at` = момент входа + `SESSION_TTL_SECONDS`; ни запросы, ни рестарт этот срок не продлевают. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `make test` | `PASS: lab2 public API scenario` | `evidence/smoke.txt` | +| Контракт и структура | `make check` | `PASS` | `evidence/check.txt` | +| Полная валидация OpenAPI | `make validate` (venv с `requirements-dev.txt`) | `PASS: full OpenAPI 3.1 and JSON Schema validation` | `evidence/validate.txt` | +| gofmt, vet, build | `gofmt -l`, `go vet ./...`, `go build ./...` | PASS | `evidence/quality.txt` | +| Docker build/run/health | `make up`, `curl /health/*` | 3 сервиса healthy, образ 7.5 MB, user=app | `evidence/docker.txt` | +| Рестарт API и всех хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; `StartedAt` у api/postgres/valkey до и после разный, volume тот же | `evidence/persistence.txt` | +| Миграции с нуля и повторно | `docker compose run --rm migrate` | `migrations up to date`, в `schema_migrations` одна строка | `evidence/migrations.txt` | +| Durability COMMIT | `SHOW` на работающем стенде | `fsync=on`, `synchronous_commit=on`, `full_page_writes=on` | `evidence/durability.txt` | +| Basic только на `/auth/sessions`, X-User-Id не даёт доступа | smoke + student-tests | Basic на `/users/me` → 401; `X-User-Id` без Bearer → 401 | `evidence/smoke.txt` | +| Пароль — KDF, токен не хранится | student-test | `$argon2id$v=19$m=19456,t=2,p=1$…`; сырого токена в БД нет | `evidence/student-tests.txt` | +| Expiry при коротком TTL | student-test, `api-short-ttl` с TTL=2 | через 2.5 с 401 и на :8081, и на основном API (кэш не продлил) | `evidence/student-tests.txt` | +| Logout и неверный пароль | student-tests | 204 → 401 → 401; неверный пароль и неизвестный пользователь получают одинаковый 401 за 37–47 мс | `evidence/student-tests.txt` | +| Гонка регистрации | 20 потоков с барьером | 1×201, 19×409, в БД одна строка | `evidence/student-tests.txt` | +| Rollback создания чата | неизвестный member | 400; строки чата нет, `xact_rollback` вырос | `evidence/student-tests.txt` | +| Отказ Valkey / PostgreSQL | `docker compose stop valkey` / `postgres` | см. раздел «Деградация» | `evidence/student-tests.txt` | +| Ограниченный пул | `tests/load.py --concurrency 32` + `pg_stat_activity` | 10 соединений при 32 клиентах; 1110 RPS, 0 ошибок | `evidence/load-32.json` | +| Публичная нагрузка | `python tests/load.py --duration 30 --concurrency 4` | 11 007 POST за 30 с, 366 RPS, 0 ошибок, p50/p95/p99 = 5.5/28.8/31.7 мс | `evidence/load.json` | +| EXPLAIN на ≥10 000 сообщений | трек 2, 110 025 сообщений | `Index Scan using messages_chat_history_idx`, 0.11 мс в одиночном прогоне (медиана 0.044 мс) | `evidence/track2/explain_production_deep_keyset.txt` | +| Трек 2 | `make track2` (`--repeat 21`) | сравнение 4 индексов и 2 видов пагинации; backup → restore → сверка | `evidence/track2/results.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Кэш Valkey (оценка 4) -## Эксперименты и отказы +### Источник истины и роль Valkey -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +PostgreSQL — источник истины для пользователей, чатов, членства, сообщений и сессий. Valkey 8.0 — вспомогательный **кэш сессий** (cache-aside). `Resolve` сначала читает `sess:` из Valkey. При промахе или ошибке он идёт в PostgreSQL (`sessions JOIN users WHERE revoked_at IS NULL AND expires_at > now()`) и заполняет кэш на `min(остаток TTL сессии, SESSION_CACHE_MAX_TTL_SECONDS)`. В записи кэша хранится `expires_at`, и приложение проверяет его даже при попадании, поэтому кэш никогда не продлевает сессию. + +Реализация: `internal/auth/sessions.go` (логика) и `internal/cache/valkey.go` (клиент). + +### Конфигурация и отключение + +- `SESSION_STORE=postgres` — Valkey не подключается, сессии читаются только из PostgreSQL; `SESSION_STORE=valkey` (по умолчанию в compose) — PostgreSQL + кэш. +- `SESSION_CACHE_MAX_TTL_SECONDS` (по умолчанию 60) — максимальный TTL записи в кэше; ограничивает окно устаревания при сбое eviction во время logout. +- `SESSION_TTL_SECONDS` (3600) — срок жизни самой сессии; `DB_MAX_CONNS` (10) — размер пула. +- Persistence Valkey **выключена** (`save ""`, `appendonly no`, `infra/valkey/valkey.conf`). Valkey не единственное хранилище сессий, поэтому после restart или `FLUSHALL` кэш просто пуст, а сессии читаются из PostgreSQL. Проверено тестом: после `FLUSHALL` токен работает, кэш заполняется снова. +- `maxmemory 64mb`, `volatile-ttl`; пароль Valkey передаётся через Docker secret. + +### Инвалидация + +Logout выполняет три шага: DEL в Valkey → `UPDATE sessions SET revoked_at` → повторный DEL. Второй DEL закрывает гонку с параллельным `Resolve`, который мог заново положить запись между первым DEL и COMMIT. Если Valkey недоступен, logout отвечает **503 ещё до изменения БД**, иначе отозванный токен мог бы остаться в кэше. Худший случай — Valkey упал между COMMIT и вторым DEL — ограничен TTL записи, то есть не больше 60 с. + +### Деградация при потере зависимостей + +- **Valkey недоступен.** `Resolve` читает из PostgreSQL, и запрос обслуживается. Таймауты клиента 0.3–0.5 с, повторов нет. Circuit breaker на 2 с: после первой ошибки следующие запросы сразу идут в БД, не дожидаясь таймаута. В тесте 8 запросов в режиме деградации заняли около 1.2 с суммарно. Readiness остаётся 200, регистрация, вход и отправка сообщений работают, logout возвращает 503. После `start valkey` logout снова отвечает 204. +- **PostgreSQL недоступен.** `/health/live` 200, `/health/ready` 503, бизнес-запросы 503. Без токена — 401. Неизвестный токен получает 503, а не 200: проверить его нельзя, доверять тоже. После `start postgres` данные на месте. +- `GET /health/ready` проверяет только PostgreSQL; Valkey некритичен. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Рестарт API + PostgreSQL + Valkey (volumes сохранены) | данные и действующая сессия сохраняются | PASS публичного теста, тот же токен, история без дублей | ~6.6 с на сценарий, потерь нет | +| `stop valkey` | обход кэша, logout отказывает безопасно | 2xx на чтение и запись, logout 503 | сразу после `start`, потерь нет | +| `FLUSHALL` | никто не разлогинен | токен работает, кэш перезаполнен | мгновенно | +| `stop postgres` | ready 503, бизнес 503, анонимы не проходят | так и есть | 2–5 с после `start`, подтверждённые данные целы | -## Ограничения и следующие шаги +## Трек 2: план запроса и восстановление -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Гипотеза.** Для запроса истории `WHERE chat_id=$1 AND (created_at,id) > cursor ORDER BY created_at, id LIMIT 51` составной индекс `(chat_id, created_at, id)` даёт постоянную стоимость для любого чата и любой глубины. Короткие индексы хороши только на части распределения размеров чатов. OFFSET-пагинация деградирует линейно при любом индексе. + +**Методика.** `scripts/track2/run_track2.py` работает в отдельном контейнере `pg-research` со свежим volume; рабочая БД стенда не затрагивается. Схема берётся из того же файла миграции, что использует приложение. +- **Данные.** Детерминированный сид `scripts/track2/seed.sql` (setseed, md5-UUID): 110 025 сообщений в 2 021 чате. + - Один чат на 60 000 сообщений — 55 % всех данных. + - 20 чатов по 1 500 сообщений и 2 000 чатов по 1–19 (медиана 10). + - Средний текст — 421 символ, таблица — 64.7 MB. +- **Перебор индексов.** Вторичные индексы `messages` пересоздаются в 4 вариантах, после каждого — `ANALYZE`. +- **Чтение.** Пять запросов, каждый через `EXPLAIN (ANALYZE, BUFFERS)` 21 раз (`--repeat 21`). Первый прогон — прогрев, в таблицу идёт медиана остальных 20. +- **Запись.** `BEGIN; EXPLAIN (ANALYZE, WAL) INSERT 20 000; ROLLBACK`, 10 прогонов, медиана 9. +- **Окружение.** PostgreSQL 17.2 с настройками по умолчанию, Docker Desktop: 22 vCPU, 15.4 GiB. + +**Чтение** (медиана, мс / буферы): + +| Индекс | big: 1-я стр. | big: keyset после 55 000 | big: OFFSET 55 000 | medium (1 500) | small (≈10) | +| --- | ---: | ---: | ---: | ---: | ---: | +| только PK | 11.87 / 6826 | 9.50 / 6768 | 48.70 / 6826 | 9.69 / 6738 | 9.67 / 6738 | +| `(chat_id)` | 13.73 / 9900 | 9.32 / 3733 | 50.23 / 9900 | 0.42 / 129 | 0.042 / 5 | +| `(created_at, id)` | 0.045 / 29 | 0.064 / 70 | 51.12 / 12977 | 0.37 / 1336 | 17.53 / 12889 | +| **`(chat_id, created_at, id)`** | **0.041 / 8** | **0.044 / 9** | 54.47 / 16054 | **0.046 / 9** | **0.036 / 6** | + +**Запись** (INSERT 20 000 строк в большой чат): + +| Индекс | Время, мс | WAL, MB | Размер индекса, MB | +| --- | ---: | ---: | ---: | +| только PK | 421 | 4.32 | 0 | +| `(chat_id)` | 483 | 5.71 | 0.82 | +| `(created_at, id)` | 498 | 5.95 | 4.27 | +| `(chat_id, created_at, id)` | 503 | 6.27 | 6.20 | + +**Выводы.** +- **Production-индекс** — единственный вариант, где все сценарии укладываются меньше чем в 0.05 мс и ≤ 10 буферов. Глубина страницы почти не влияет: 0.041 мс на первой странице и 0.044 мс после 55 000 сообщений. План — `Limit -> Index Scan using messages_chat_history_idx`, без Sort. +- **Неравномерность размеров ломает короткие индексы.** + - `(chat_id)` хорош для маленьких чатов, но на большом даёт Bitmap Scan + Sort — примерно в 200 раз медленнее. + - `(created_at, id)` — наоборот: для чата из 10 сообщений планировщик уходит в Seq Scan (17.5 мс, 12 889 буферов). + - На средних значениях этого не видно. +- **OFFSET не спасает ни один индекс.** Пропущенные 55 000 строк всё равно читаются, и keyset больше чем в 1000 раз быстрее (0.044 против 54.47 мс). Поэтому API использует курсор `(created_at, id)`. +- **Цена записи.** Устойчиво измеряется объёмом WAL и размером индекса: составной индекс даёт +46 % WAL на вставку (6.27 против 4.32 MB на 20 000 строк) и весит 6.2 MB, около 10 % размера таблицы. Время пакетной вставки ненадёжно: в последнем прогоне оно росло вместе с числом индексов (421 → 503 мс, +20 %), но в предыдущих прогонах на той же машине порядок вариантов был другим, а абсолютные значения отличались почти вдвое (855–1010 мс). Поэтому разницу во времени я не считаю доказанной. Отдельный индекс `(chat_id)` становится лишним: составной покрывает его как префикс. + +**Backup → восстановление в новое хранилище → сверка:** + +| Этап | Время, с | +| --- | ---: | +| `pg_dump -Fc` (дамп 6.9 MB) | 0.85 | +| копирование дампа на хост (`backups/`, в `.gitignore`) | 0.22 | +| новый контейнер `pg-restore` с **пустым** volume, копирование дампа внутрь | 0.69 | +| `pg_restore -j 4 --exit-on-error` (данные, ключи, индексы) | 1.03 | +| `ANALYZE` | 0.50 | +| **Итого RTO после появления пустого сервера** | **2.21** | + +Сверка «источник на момент бэкапа» против восстановленной копии — **12 из 12 проверок совпали**: +- количество строк во всех таблицах и `schema_migrations`; +- порядконезависимая контрольная сумма всех 110 025 сообщений; +- md5 счётчиков по каждому чату; +- md5 полной истории большого, среднего и маленького чата; +- md5 выборки из 200 сообщений; +- список индексов. + +После восстановления запрос истории снова идёт через `messages_chat_history_idx`. + +**Граница потери данных.** Сразу после дампа в источник записывается ещё одно сообщение, и в восстановленной копии его нет. `pg_dump` фиксирует снимок на момент старта, поэтому RPO равен интервалу между бэкапами. Уменьшить его можно только архивацией WAL (PITR) или репликой; в стенде это не реализовано. + +### Ограничения + +- **Logout зависит от Valkey.** При недоступном Valkey возвращается 503 — сознательный выбор безопасности. +- **Окно устаревания кэша.** Если Valkey упал между COMMIT отзыва и вторым DEL, токен может приниматься до `SESSION_CACHE_MAX_TTL_SECONDS` (60 с). +- **Кэшируются только сессии.** Списки чатов и история всегда читаются из PostgreSQL; кэш истории потребовал бы инвалидации на каждую запись. +- **Измерения трека 2 сделаны на тёплом кэше.** Таблица помещается в `shared_buffers`, замер в одном окружении и без параллельной нагрузки. Холодное чтение увеличит разрыв между Seq Scan и Index Scan. +- **Размер дампа занижен.** Синтетический текст из повторов md5 хорошо сжимается: 64.7 MB → 6.9 MB. На реальных текстах дамп и время восстановления будут больше. +- **Бэкап только логический (`pg_dump`).** Нет WAL-архива и расписания бэкапов. +- **Нет rate limiting на `POST /auth/sessions`.** Argon2id замедляет подбор пароля, но не заменяет лимит попыток. +- **HTTP без TLS.** Допустимо только для локального изолированного стенда: Basic — это кодирование, а не шифрование. +- **COMMIT гарантирует сохранность только на одном диске.** При `fsync=on` и `synchronous_commit=on` ответ 201 отправляется после записи WAL на диск volume; в Docker Desktop это виртуальный диск VM. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..992e5b56 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,142 @@ +name: mtusi-lab2 + +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_STORE: ${SESSION_STORE:-valkey} + SESSION_CACHE_MAX_TTL_SECONDS: ${SESSION_CACHE_MAX_TTL_SECONDS:-60} + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + +services: + postgres: + image: postgres:17.2-alpine3.21 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/pg_password + secrets: [pg_password] + volumes: + - pgdata:/var/lib/postgresql/data + networks: [backend] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"] + interval: 2s + timeout: 3s + retries: 30 + stop_grace_period: 30s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.0.2-alpine3.21 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + volumes: + - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + secrets: [valkey_auth] + networks: [backend] + healthcheck: + test: ["CMD-SHELL", "valkey-cli ping | grep -qE 'PONG|NOAUTH'"] + interval: 2s + timeout: 3s + retries: 30 + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab2 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [backend] + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + image: mtusi-messenger:lab2 + environment: + <<: *api-env + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url, cache_url] + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + networks: [backend] + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + valkey: + condition: service_healthy + stop_grace_period: 10s + restart: unless-stopped + + api-short-ttl: + profiles: [ttl] + image: mtusi-messenger:lab2 + environment: + <<: *api-env + SESSION_TTL_SECONDS: "2" + SESSION_CACHE_MAX_TTL_SECONDS: "60" + secrets: [database_url, cache_url] + ports: + - "127.0.0.1:8081:8080" + networks: [backend] + depends_on: + migrate: + condition: service_completed_successfully + restart: unless-stopped + + pg-research: + profiles: [research] + image: postgres:17.2-alpine3.21 + environment: &pg-env + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/pg_password + secrets: [pg_password] + shm_size: 256mb + volumes: + - pgresearch:/var/lib/postgresql/data + - ./src/internal/db/migrations:/migrations:ro + networks: [research] + healthcheck: &pg-health + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"] + interval: 2s + timeout: 3s + retries: 30 + + pg-restore: + profiles: [research] + image: postgres:17.2-alpine3.21 + environment: *pg-env + secrets: [pg_password] + shm_size: 256mb + volumes: + - pgrestore:/var/lib/postgresql/data + networks: [research] + healthcheck: *pg-health + +networks: + backend: {} + research: {} + +volumes: + pgdata: {} + pgresearch: {} + pgrestore: {} + +secrets: + pg_password: + file: ./secrets/pg_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + valkey_auth: + file: ./secrets/valkey_auth.conf diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..f756e410 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,3 @@ +# check — 2026-10-07T20:31:08Z, branch solution/BVT2304-korablev +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/docker.txt b/evidence/docker.txt new file mode 100644 index 00000000..2cf6deab --- /dev/null +++ b/evidence/docker.txt @@ -0,0 +1,12 @@ +# docker — 2026-10-07T20:31:13Z, branch solution/BVT2304-korablev +$ docker compose ps / images / health +SERVICE IMAGE STATUS +api mtusi-messenger:lab2 Up 13 minutes (healthy) +postgres postgres:17.2-alpine3.21 Up 9 minutes (healthy) +valkey valkey/valkey:8.0.2-alpine3.21 Up 10 minutes (healthy) + +image mtusi-messenger:lab2 size=7463250 user=app + +$ curl /health/live /health/ready +{"status":"ok"} +{"status":"ok"} diff --git a/evidence/durability.txt b/evidence/durability.txt new file mode 100644 index 00000000..9d5a3bef --- /dev/null +++ b/evidence/durability.txt @@ -0,0 +1,6 @@ +# durability — 2026-10-07T20:31:36Z, branch solution/BVT2304-korablev +$ SHOW fsync/synchronous_commit/... +fsync=on +full_page_writes=on +synchronous_commit=on +wal_level=replica diff --git a/evidence/load-32.json b/evidence/load-32.json new file mode 100644 index 00000000..3f097848 --- /dev/null +++ b/evidence/load-32.json @@ -0,0 +1,17 @@ +{ + "lab": 2, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.224, + "concurrency": 32, + "successful_requests": 11354, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1110.576, + "successful_latency_ms": { + "p50": 27.077, + "p95": 44.687, + "p99": 53.944 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..a979680b --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 2, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.04, + "concurrency": 4, + "successful_requests": 11007, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 366.408, + "successful_latency_ms": { + "p50": 5.504, + "p95": 28.791, + "p99": 31.703 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/migrations.txt b/evidence/migrations.txt new file mode 100644 index 00000000..32ef1a9b --- /dev/null +++ b/evidence/migrations.txt @@ -0,0 +1,5 @@ +# migrations — 2026-10-07T20:31:14Z, branch solution/BVT2304-korablev +$ docker compose run --rm migrate (повторный запуск) +{"level":"INFO","msg":"migrations up to date","total":1} +$ select * from schema_migrations +1|0001_init.sql diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..2748cee1 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,16 @@ +# persistence — 2026-10-07T20:31:17Z, branch solution/BVT2304-korablev +$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +## StartedAt BEFORE +api 2026-10-07T20:17:23.346707028Z +postgres 2026-10-07T20:21:18.602507236Z +valkey 2026-10-07T20:21:00.713703458Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +## StartedAt AFTER (все три перезапущены) +api 2026-10-07T20:31:26.847094604Z +postgres 2026-10-07T20:31:23.343062647Z +valkey 2026-10-07T20:31:23.29843169Z +## volume +mtusi-lab2_pgdata +## restart.sh напрямую (Git Bash) +rc=0 diff --git a/evidence/quality.txt b/evidence/quality.txt new file mode 100644 index 00000000..fe784b3a --- /dev/null +++ b/evidence/quality.txt @@ -0,0 +1,6 @@ +# quality — 2026-10-07T20:31:10Z, branch solution/BVT2304-korablev +$ gofmt -l . && go vet ./... && go build ./... +gofmt -l: [] +go vet: ok +go build: ok +go version go1.27.1 windows/amd64 diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..9b21bb0f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,3 @@ +# smoke — 2026-10-07T20:31:08Z, branch solution/BVT2304-korablev +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.json b/evidence/student-tests.json new file mode 100644 index 00000000..b1ef1019 --- /dev/null +++ b/evidence/student-tests.json @@ -0,0 +1,74 @@ +[ + { + "test": "wrong password / unknown user -> одинаковый 401 Basic", + "status": "PASS", + "seconds": 0.47, + "note": "known=23ms unknown=22ms" + }, + { + "test": "пароль: 12..128 code points, двоеточие допустимо", + "status": "PASS", + "seconds": 0.12, + "note": "" + }, + { + "test": "logout: 204, затем 401 на запрос и повторный logout", + "status": "PASS", + "seconds": 0.18, + "note": "" + }, + { + "test": "токен непрозрачный, в БД только SHA-256, пароль — Argon2id", + "status": "PASS", + "seconds": 0.9, + "note": "" + }, + { + "test": "expiry при SESSION_TTL_SECONDS=2 (api-short-ttl :8081)", + "status": "PASS", + "seconds": 5.67, + "note": "" + }, + { + "test": "гонка регистрации одного username: ровно один 201", + "status": "PASS", + "seconds": 0.73, + "note": "201x1, 409x19" + }, + { + "test": "rollback составной операции создания чата", + "status": "PASS", + "seconds": 3.56, + "note": "xact_rollback 164 -> 167" + }, + { + "test": "параметризованные запросы: SQL-символы сохраняются как текст", + "status": "PASS", + "seconds": 0.22, + "note": "" + }, + { + "test": "пагинация 250 сообщений: без потерь и повторов, курсор чужого чата -> 400", + "status": "PASS", + "seconds": 4.27, + "note": "7 pages" + }, + { + "test": "Valkey реально кэширует сессии (TTL <= 60s), FLUSHALL не разлогинивает", + "status": "PASS", + "seconds": 1.76, + "note": "keys=15 ttl=30s" + }, + { + "test": "отказ Valkey: API обходит кэш, logout -> 503 (не оставляет отозванный токен в кэше)", + "status": "PASS", + "seconds": 2.42, + "note": "degraded requests took 1.10s total" + }, + { + "test": "отказ PostgreSQL: ready -> 503, бизнес-запросы 503, аноним/неизвестный токен не проходят", + "status": "PASS", + "seconds": 17.09, + "note": "" + } +] diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..297c248e --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,16 @@ +# student-tests — python student-tests/run_tests.py --json evidence/student-tests.json + +PASS 0.47s wrong password / unknown user -> одинаковый 401 Basic [known=23ms unknown=22ms] +PASS 0.12s пароль: 12..128 code points, двоеточие допустимо +PASS 0.18s logout: 204, затем 401 на запрос и повторный logout +PASS 0.90s токен непрозрачный, в БД только SHA-256, пароль — Argon2id +PASS 5.67s expiry при SESSION_TTL_SECONDS=2 (api-short-ttl :8081) +PASS 0.73s гонка регистрации одного username: ровно один 201 [201x1, 409x19] +PASS 3.56s rollback составной операции создания чата [xact_rollback 164 -> 167] +PASS 0.22s параметризованные запросы: SQL-символы сохраняются как текст +PASS 4.27s пагинация 250 сообщений: без потерь и повторов, курсор чужого чата -> 400 [7 pages] +PASS 1.76s Valkey реально кэширует сессии (TTL <= 60s), FLUSHALL не разлогинивает [keys=15 ttl=30s] +PASS 2.42s отказ Valkey: API обходит кэш, logout -> 503 (не оставляет отозванный токен в кэше) [degraded requests took 1.10s total] +PASS 17.09s отказ PostgreSQL: ready -> 503, бизнес-запросы 503, аноним/неизвестный токен не проходят + +12/12 passed diff --git a/evidence/track2/explain_pk_only_deep_keyset.txt b/evidence/track2/explain_pk_only_deep_keyset.txt new file mode 100644 index 00000000..a9fcb033 --- /dev/null +++ b/evidence/track2/explain_pk_only_deep_keyset.txt @@ -0,0 +1 @@ +see results.json -> read.pk_only diff --git a/evidence/track2/explain_production_deep_keyset.txt b/evidence/track2/explain_production_deep_keyset.txt new file mode 100644 index 00000000..3159ab5a --- /dev/null +++ b/evidence/track2/explain_production_deep_keyset.txt @@ -0,0 +1,9 @@ +Limit (cost=0.42..167.46 rows=51 width=462) (actual time=0.032..0.057 rows=51 loops=1) + Buffers: shared hit=5 read=4 + -> Index Scan using messages_chat_history_idx on messages (cost=0.42..18702.46 rows=5710 width=462) (actual time=0.030..0.050 rows=51 loops=1) + Index Cond: ((chat_id = '64fbbeee-3c1b-b5be-9d4f-5bd05fe83171'::uuid) AND (ROW(created_at, id) > ROW('2026-06-16 00:00:00.729+00'::timestamp with time zone, '816028d8-560b-edb9-b9ae-2bfe37b74505'::uuid))) + Buffers: shared hit=5 read=4 +Planning: + Buffers: shared hit=151 read=1 +Planning Time: 1.206 ms +Execution Time: 0.107 ms diff --git a/evidence/track2/results.json b/evidence/track2/results.json new file mode 100644 index 00000000..5c143630 --- /dev/null +++ b/evidence/track2/results.json @@ -0,0 +1,361 @@ +{ + "started_at": "2026-10-07T21:18:58+00:00", + "environment": { + "docker_cpus": 22, + "docker_mem_gib": 15.4, + "docker_os": "Docker Desktop", + "postgres": "17.2", + "shared_buffers": "128MB", + "work_mem": "4MB" + }, + "dataset": { + "messages": 110025, + "chats": 2021, + "users": 2000, + "max_chat": 60000, + "median_chat": 10, + "avg_text_chars": 421, + "max_text_chars": 1984, + "table_mb": 64.7, + "seed_seconds": 4.1 + }, + "read": { + "pk_only": { + "big_first_page": { + "exec_ms_median": 11.866, + "exec_ms_min": 10.706, + "exec_ms_max": 13.437, + "planning_ms_median": 0.129, + "rows": 51, + "shared_hit": 6826, + "shared_read": 0, + "plan": "Limit -> Gather Merge -> Sort -> Seq Scan", + "runs": 20 + }, + "big_deep_keyset": { + "exec_ms_median": 9.495, + "exec_ms_min": 8.872, + "exec_ms_max": 10.584, + "planning_ms_median": 0.114, + "rows": 51, + "shared_hit": 6768, + "shared_read": 0, + "plan": "Limit -> Gather Merge -> Sort -> Seq Scan", + "runs": 20 + }, + "big_deep_offset": { + "exec_ms_median": 48.697, + "exec_ms_min": 45.498, + "exec_ms_max": 54.401, + "planning_ms_median": 0.124, + "rows": 51, + "shared_hit": 6826, + "shared_read": 0, + "plan": "Limit -> Gather Merge -> Sort -> Seq Scan", + "runs": 20 + }, + "medium_first_page": { + "exec_ms_median": 9.692, + "exec_ms_min": 8.654, + "exec_ms_max": 11.999, + "planning_ms_median": 0.126, + "rows": 51, + "shared_hit": 6738, + "shared_read": 0, + "plan": "Limit -> Sort -> Seq Scan", + "runs": 20 + }, + "small_first_page": { + "exec_ms_median": 9.674, + "exec_ms_min": 8.77, + "exec_ms_max": 10.79, + "planning_ms_median": 0.137, + "rows": 14, + "shared_hit": 6738, + "shared_read": 0, + "plan": "Limit -> Sort -> Seq Scan", + "runs": 20 + } + }, + "chat_id": { + "big_first_page": { + "exec_ms_median": 13.729, + "exec_ms_min": 13.19, + "exec_ms_max": 14.689, + "planning_ms_median": 0.124, + "rows": 51, + "shared_hit": 9900, + "shared_read": 0, + "plan": "Limit -> Gather Merge -> Sort -> Seq Scan", + "runs": 20 + }, + "big_deep_keyset": { + "exec_ms_median": 9.317, + "exec_ms_min": 8.088, + "exec_ms_max": 10.636, + "planning_ms_median": 0.137, + "rows": 51, + "shared_hit": 3733, + "shared_read": 0, + "plan": "Limit -> Sort -> Bitmap Heap Scan -> Bitmap Index Scan(t2_ix)", + "runs": 20 + }, + "big_deep_offset": { + "exec_ms_median": 50.228, + "exec_ms_min": 48.063, + "exec_ms_max": 69.066, + "planning_ms_median": 0.146, + "rows": 51, + "shared_hit": 9900, + "shared_read": 0, + "plan": "Limit -> Gather Merge -> Sort -> Seq Scan", + "runs": 20 + }, + "medium_first_page": { + "exec_ms_median": 0.417, + "exec_ms_min": 0.305, + "exec_ms_max": 0.608, + "planning_ms_median": 0.057, + "rows": 51, + "shared_hit": 129, + "shared_read": 0, + "plan": "Limit -> Sort -> Bitmap Heap Scan -> Bitmap Index Scan(t2_ix)", + "runs": 20 + }, + "small_first_page": { + "exec_ms_median": 0.042, + "exec_ms_min": 0.034, + "exec_ms_max": 0.084, + "planning_ms_median": 0.054, + "rows": 14, + "shared_hit": 5, + "shared_read": 0, + "plan": "Limit -> Sort -> Index Scan(t2_ix)", + "runs": 20 + } + }, + "created_at_id": { + "big_first_page": { + "exec_ms_median": 0.045, + "exec_ms_min": 0.038, + "exec_ms_max": 0.094, + "planning_ms_median": 0.051, + "rows": 51, + "shared_hit": 29, + "shared_read": 0, + "plan": "Limit -> Index Scan(t2_ix)", + "runs": 20 + }, + "big_deep_keyset": { + "exec_ms_median": 0.064, + "exec_ms_min": 0.05, + "exec_ms_max": 0.096, + "planning_ms_median": 0.055, + "rows": 51, + "shared_hit": 70, + "shared_read": 0, + "plan": "Limit -> Index Scan(t2_ix)", + "runs": 20 + }, + "big_deep_offset": { + "exec_ms_median": 51.115, + "exec_ms_min": 49.001, + "exec_ms_max": 56.033, + "planning_ms_median": 0.136, + "rows": 51, + "shared_hit": 12977, + "shared_read": 0, + "plan": "Limit -> Gather Merge -> Sort -> Seq Scan", + "runs": 20 + }, + "medium_first_page": { + "exec_ms_median": 0.368, + "exec_ms_min": 0.307, + "exec_ms_max": 0.902, + "planning_ms_median": 0.04, + "rows": 51, + "shared_hit": 1336, + "shared_read": 0, + "plan": "Limit -> Index Scan(t2_ix)", + "runs": 20 + }, + "small_first_page": { + "exec_ms_median": 17.53, + "exec_ms_min": 16.59, + "exec_ms_max": 18.85, + "planning_ms_median": 0.17, + "rows": 14, + "shared_hit": 12889, + "shared_read": 0, + "plan": "Limit -> Sort -> Seq Scan", + "runs": 20 + } + }, + "chat_created_id": { + "big_first_page": { + "exec_ms_median": 0.041, + "exec_ms_min": 0.033, + "exec_ms_max": 0.078, + "planning_ms_median": 0.044, + "rows": 51, + "shared_hit": 8, + "shared_read": 0, + "plan": "Limit -> Index Scan(messages_chat_history_idx)", + "runs": 20 + }, + "big_deep_keyset": { + "exec_ms_median": 0.044, + "exec_ms_min": 0.037, + "exec_ms_max": 0.069, + "planning_ms_median": 0.045, + "rows": 51, + "shared_hit": 9, + "shared_read": 0, + "plan": "Limit -> Index Scan(messages_chat_history_idx)", + "runs": 20 + }, + "big_deep_offset": { + "exec_ms_median": 54.471, + "exec_ms_min": 50.873, + "exec_ms_max": 56.894, + "planning_ms_median": 0.132, + "rows": 51, + "shared_hit": 16054, + "shared_read": 0, + "plan": "Limit -> Gather Merge -> Sort -> Seq Scan", + "runs": 20 + }, + "medium_first_page": { + "exec_ms_median": 0.046, + "exec_ms_min": 0.033, + "exec_ms_max": 0.069, + "planning_ms_median": 0.05, + "rows": 51, + "shared_hit": 9, + "shared_read": 0, + "plan": "Limit -> Index Scan(messages_chat_history_idx)", + "runs": 20 + }, + "small_first_page": { + "exec_ms_median": 0.036, + "exec_ms_min": 0.028, + "exec_ms_max": 0.047, + "planning_ms_median": 0.041, + "rows": 14, + "shared_hit": 6, + "shared_read": 0, + "plan": "Limit -> Index Scan(messages_chat_history_idx)", + "runs": 20 + } + } + }, + "write": { + "pk_only": { + "insert_20k_ms_median": 420.9, + "wal_bytes_median": 4531341, + "runs": 9, + "secondary_index_mb": 0.0 + }, + "chat_id": { + "insert_20k_ms_median": 482.5, + "wal_bytes_median": 5989805, + "runs": 9, + "secondary_index_mb": 0.82 + }, + "created_at_id": { + "insert_20k_ms_median": 497.6, + "wal_bytes_median": 6234752, + "runs": 9, + "secondary_index_mb": 4.27 + }, + "chat_created_id": { + "insert_20k_ms_median": 503.4, + "wal_bytes_median": 6570551, + "runs": 9, + "secondary_index_mb": 6.2 + } + }, + "backup_restore": { + "backup_taken_at": "2026-10-07 21:19:41.540149+00", + "dump_seconds": 0.85, + "dump_mb": 6.9, + "copy_out_seconds": 0.22, + "copy_in_seconds": 0.69, + "pg_restore_seconds": 1.03, + "analyze_seconds": 0.5, + "total_recovery_seconds": 2.21 + }, + "verification": { + "checks": { + "users": { + "source": "2000", + "restored": "2000", + "match": true + }, + "chats": { + "source": "2021", + "restored": "2021", + "match": true + }, + "chat_members": { + "source": "5250", + "restored": "5250", + "match": true + }, + "messages": { + "source": "110025", + "restored": "110025", + "match": true + }, + "schema_migrations": { + "source": "1", + "restored": "1", + "match": true + }, + "messages_checksum": { + "source": "63211757690929886368194", + "restored": "63211757690929886368194", + "match": true + }, + "per_chat_counts_md5": { + "source": "c9923d86da3e7be8ef64323c8917e869", + "restored": "c9923d86da3e7be8ef64323c8917e869", + "match": true + }, + "big_chat_history_md5": { + "source": "e21b19ba52dca7b57c2032c5c948f9ba", + "restored": "e21b19ba52dca7b57c2032c5c948f9ba", + "match": true + }, + "medium_chat_history_md5": { + "source": "f07aa1e21b5268f1321fedf6cfb6e64a", + "restored": "f07aa1e21b5268f1321fedf6cfb6e64a", + "match": true + }, + "small_chat_history_md5": { + "source": "c58b6535cdbb7b72cc43b895cea34de6", + "restored": "c58b6535cdbb7b72cc43b895cea34de6", + "match": true + }, + "sample_200_md5": { + "source": "e2088ba6aeeb4c1a89beb667f76955e9", + "restored": "e2088ba6aeeb4c1a89beb667f76955e9", + "match": true + }, + "indexes": { + "source": "chat_members_pkey,chat_members_position_key,chat_members_user_idx,chats_pkey,messages_chat_history_idx,messages_pkey,schema_migrations_pkey,sessions_expires_idx,sessions_pkey,sessions_user_idx,users_pkey,users_username_key", + "restored": "chat_members_pkey,chat_members_position_key,chat_members_user_idx,chats_pkey,messages_chat_history_idx,messages_pkey,schema_migrations_pkey,sessions_expires_idx,sessions_pkey,sessions_user_idx,users_pkey,users_username_key", + "match": true + } + }, + "all_match": true, + "post_backup_write_in_source": true, + "post_backup_write_in_restored": false, + "restored_history_plan": [ + "Limit (cost=0.42..137.32 rows=51 width=462)", + " -> Index Scan using messages_chat_history_idx on messages (cost=0.42..14831.87 rows=5525 width=462)", + " Index Cond: ((chat_id = '64fbbeee-3c1b-b5be-9d4f-5bd05fe83171'::uuid) AND (ROW(created_at, id) > ROW('2026-06-16 00:00:00.729+00'::timestamp with time zone, '816028d8-560b-edb9-b9ae-2bfe37b74505'::uuid)))" + ] + }, + "finished_at": "2026-10-07T21:19:53+00:00" +} diff --git a/evidence/validate.txt b/evidence/validate.txt new file mode 100644 index 00000000..57535e22 --- /dev/null +++ b/evidence/validate.txt @@ -0,0 +1,3 @@ +# validate — 2026-10-07T21:05:25Z +$ .venv/Scripts/python scripts/validate_openapi.py (requirements-dev.txt: openapi-spec-validator==0.7.2) +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 00000000..6e50813e --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,8 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +save "" +appendonly no +maxmemory 64mb +maxmemory-policy volatile-ttl +include /run/secrets/valkey_auth diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..c154be7a --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +import secrets +import shutil +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / 'secrets' + + +def write(name, value): + path = SECRETS / name + if path.exists(): + print('keep secrets/' + name) + return path.read_text().strip() + path.write_text(value + '\n', newline='\n') + print('created secrets/' + name) + return value + + +def main(): + SECRETS.mkdir(exist_ok=True) + pg = write('pg_password', secrets.token_urlsafe(24)) + write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg) + vk_conf = SECRETS / 'valkey_auth.conf' + if vk_conf.exists(): + vk = vk_conf.read_text().split()[-1] + print('keep secrets/valkey_auth.conf') + else: + vk = secrets.token_urlsafe(24) + write('valkey_auth.conf', 'requirepass ' + vk) + write('cache_url', 'redis://:%s@valkey:6379/0' % vk) + env = ROOT / '.env' + if not env.exists(): + shutil.copy(ROOT / '.env.example', env) + print('created .env from .env.example') + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..6be47c2c --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,5 @@ +@echo off +cd /d "%~dp0.." +docker compose stop api || exit /b 1 +docker compose restart postgres valkey || exit /b 1 +docker compose start api || exit /b 1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a1375680 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose stop api +docker compose restart postgres valkey +docker compose start api diff --git a/scripts/track2/run_track2.py b/scripts/track2/run_track2.py new file mode 100644 index 00000000..c5a8f588 --- /dev/null +++ b/scripts/track2/run_track2.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +import argparse +import datetime as dt +import json +import statistics +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +OUT = ROOT / 'evidence' / 'track2' +PROJECT = 'mtusi-lab2' +SRC, DST = 'pg-research', 'pg-restore' + +BIG = "md5('chat1')::uuid" +MEDIUM = "md5('chat5')::uuid" +SMALL = "md5('chat500')::uuid" +COLS = 'id, chat_id, sender_id, text, created_at' + +INDEX_CONFIGS = { + 'pk_only': [], + 'chat_id': ['CREATE INDEX t2_ix ON messages (chat_id)'], + 'created_at_id': ['CREATE INDEX t2_ix ON messages (created_at, id)'], + 'chat_created_id': ['CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id)'], +} +PRODUCTION = 'chat_created_id' + + +def log(msg): + print('[%s] %s' % (dt.datetime.now().strftime('%H:%M:%S'), msg), flush=True) + + +def sh(args, input_text=None, check=True): + p = subprocess.run(args, cwd=ROOT, input=input_text, capture_output=True, text=True, encoding='utf-8') + if check and p.returncode != 0: + raise SystemExit('command failed: %s\n%s' % (' '.join(args[:6]), p.stderr[-2000:])) + return p + + +def compose(*args, **kw): + return sh(['docker', 'compose', '--profile', 'research', *args], **kw) + + +def psql(service, *sql, input_text=None): + args = ['exec', '-T', service, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAX', '-v', 'ON_ERROR_STOP=1'] + for s in sql: + args += ['-c', s] + return compose(*args, input_text=input_text).stdout.strip() + + +def timed(fn): + t = time.perf_counter() + r = fn() + return r, time.perf_counter() - t + + +def wait_healthy(service, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + p = compose('exec', '-T', service, 'pg_isready', '-U', 'messenger', '-d', 'messenger', '-h', '127.0.0.1', check=False) + if p.returncode == 0: + q = compose('exec', '-T', service, 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT 1', check=False) + if q.returncode == 0 and q.stdout.strip() == '1': + return + time.sleep(1) + raise SystemExit(service + ' not healthy') + + +def fresh(service): + compose('rm', '-sf', service) + sh(['docker', 'volume', 'rm', '-f', '%s_%s' % (PROJECT, service.replace('-', ''))], check=False) + compose('up', '-d', service) + wait_healthy(service) + + +def walk(plan, acc): + name = plan['Node Type'] + if plan.get('Index Name'): + name += '(' + plan['Index Name'] + ')' + acc.append(name) + for child in plan.get('Plans', []): + walk(child, acc) + return acc + + +def explain_many(sql, repeat): + stmt = 'EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + sql + parts = [] + for _ in range(repeat): + parts += [stmt, r'\echo @@@'] + raw = psql(SRC, *parts) + runs = [json.loads(chunk)[0] for chunk in raw.split('@@@') if chunk.strip()] + measured = runs[1:] + last = measured[-1] + top = last['Plan'] + return { + 'exec_ms_median': round(statistics.median(r['Execution Time'] for r in measured), 3), + 'exec_ms_min': round(min(r['Execution Time'] for r in measured), 3), + 'exec_ms_max': round(max(r['Execution Time'] for r in measured), 3), + 'planning_ms_median': round(statistics.median(r['Planning Time'] for r in measured), 3), + 'rows': top.get('Actual Rows'), + 'shared_hit': top.get('Shared Hit Blocks', 0), + 'shared_read': top.get('Shared Read Blocks', 0), + 'plan': ' -> '.join(walk(top, [])), + 'runs': len(measured), + } + + +def queries(cursor_ts, cursor_id): + return { + 'big_first_page': "SELECT %s FROM messages WHERE chat_id = %s ORDER BY created_at, id LIMIT 51" % (COLS, BIG), + 'big_deep_keyset': "SELECT %s FROM messages WHERE chat_id = %s AND (created_at, id) > ('%s', '%s') ORDER BY created_at, id LIMIT 51" % (COLS, BIG, cursor_ts, cursor_id), + 'big_deep_offset': "SELECT %s FROM messages WHERE chat_id = %s ORDER BY created_at, id OFFSET 55000 LIMIT 51" % (COLS, BIG), + 'medium_first_page': "SELECT %s FROM messages WHERE chat_id = %s ORDER BY created_at, id LIMIT 51" % (COLS, MEDIUM), + 'small_first_page': "SELECT %s FROM messages WHERE chat_id = %s ORDER BY created_at, id LIMIT 51" % (COLS, SMALL), + } + + +INSERT_20K = ("INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "SELECT md5('ins' || j)::uuid, " + BIG + ", md5('user' || ((7 + j % 50) % 2000 + 1))::uuid, " + "md5(j::text), timestamptz '2026-07-01Z' + j * interval '1 ms' FROM generate_series(1, 20000) j") + + +def apply_config(name): + ddl = ['DROP INDEX IF EXISTS messages_chat_history_idx', 'DROP INDEX IF EXISTS t2_ix'] + INDEX_CONFIGS[name] + ['ANALYZE messages'] + psql(SRC, *ddl) + + +def write_cost(repeat): + parts = [] + for _ in range(repeat): + parts += ['BEGIN', 'EXPLAIN (ANALYZE, BUFFERS, WAL, FORMAT JSON) ' + INSERT_20K, 'ROLLBACK', r'\echo @@@'] + raw = psql(SRC, *parts) + runs = [] + for chunk in raw.split('@@@'): + chunk = chunk.strip() + if not chunk: + continue + body = chunk[chunk.index('['):chunk.rindex(']') + 1] + runs.append(json.loads(body)[0]) + measured = runs[1:] + return { + 'insert_20k_ms_median': round(statistics.median(r['Execution Time'] for r in measured), 1), + 'wal_bytes_median': int(statistics.median(r['Plan'].get('WAL Bytes', 0) for r in measured)), + 'runs': len(measured), + } + + +VERIFY = { + 'users': 'SELECT count(*) FROM users', + 'chats': 'SELECT count(*) FROM chats', + 'chat_members': 'SELECT count(*) FROM chat_members', + 'messages': 'SELECT count(*) FROM messages', + 'schema_migrations': "SELECT coalesce(string_agg(version::text, ',' ORDER BY version), '') FROM schema_migrations", + 'messages_checksum': "SELECT sum(('x' || substr(md5(id::text || chat_id::text || sender_id::text || text || created_at::text), 1, 15))::bit(60)::bigint) FROM messages", + 'per_chat_counts_md5': "SELECT md5(string_agg(chat_id::text || ':' || n, ',' ORDER BY chat_id)) FROM (SELECT chat_id, count(*) n FROM messages GROUP BY chat_id) t", + 'big_chat_history_md5': "SELECT md5(string_agg(id::text || '|' || text || '|' || created_at::text, E'\\n' ORDER BY created_at, id)) FROM messages WHERE chat_id = " + BIG, + 'medium_chat_history_md5': "SELECT md5(string_agg(id::text || '|' || text || '|' || created_at::text, E'\\n' ORDER BY created_at, id)) FROM messages WHERE chat_id = " + MEDIUM, + 'small_chat_history_md5': "SELECT md5(string_agg(id::text || '|' || text || '|' || created_at::text, E'\\n' ORDER BY created_at, id)) FROM messages WHERE chat_id = " + SMALL, + 'sample_200_md5': "SELECT md5(string_agg(m.id::text || '|' || m.text, ',' ORDER BY m.id)) FROM messages m WHERE m.id IN (SELECT md5('msg1:' || j)::uuid FROM generate_series(300, 60000, 300) j)", + 'indexes': "SELECT string_agg(indexname, ',' ORDER BY indexname) FROM pg_indexes WHERE schemaname = 'public'", +} + + +def verify(service): + return {k: psql(service, q) for k, q in VERIFY.items()} + + +def main(): + p = argparse.ArgumentParser(description='Трек 2: план запроса истории и backup -> restore -> сверка') + p.add_argument('--repeat', type=int, default=7, help='прогонов на замер (первый — прогрев, отбрасывается)') + a = p.parse_args() + if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') + OUT.mkdir(parents=True, exist_ok=True) + result = {'started_at': dt.datetime.now(dt.timezone.utc).isoformat(timespec='seconds')} + + info = sh(['docker', 'info', '--format', '{{.NCPU}} {{.MemTotal}} {{.OperatingSystem}}']).stdout.split(maxsplit=2) + result['environment'] = {'docker_cpus': int(info[0]), 'docker_mem_gib': round(int(info[1]) / 2**30, 1), 'docker_os': info[2].strip()} + + log('1/6 fresh research database + migrations') + fresh(SRC) + psql(SRC, r'\i /migrations/0001_init.sql', + "CREATE TABLE IF NOT EXISTS schema_migrations (version integer PRIMARY KEY, name text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())", + "INSERT INTO schema_migrations (version, name) VALUES (1, '0001_init.sql')") + result['environment'].update({ + 'postgres': psql(SRC, 'SHOW server_version'), + 'shared_buffers': psql(SRC, 'SHOW shared_buffers'), + 'work_mem': psql(SRC, 'SHOW work_mem'), + }) + + log('2/6 seeding synthetic data (≈110k messages)') + seed = (ROOT / 'scripts' / 'track2' / 'seed.sql').read_text(encoding='utf-8') + _, seed_s = timed(lambda: psql(SRC, input_text=seed)) + dist = psql(SRC, "SELECT json_build_object(" + "'messages', (SELECT count(*) FROM messages)," + "'chats', (SELECT count(*) FROM chats)," + "'users', (SELECT count(*) FROM users)," + "'max_chat', (SELECT max(n) FROM (SELECT count(*) n FROM messages GROUP BY chat_id) t)," + "'median_chat', (SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY n) FROM (SELECT count(*) n FROM messages GROUP BY chat_id) t)," + "'avg_text_chars', (SELECT round(avg(char_length(text))) FROM messages)," + "'max_text_chars', (SELECT max(char_length(text)) FROM messages)," + "'table_mb', round(pg_total_relation_size('messages') / 1048576.0, 1))") + result['dataset'] = json.loads(dist) + result['dataset']['seed_seconds'] = round(seed_s, 1) + log(' dataset: %s' % result['dataset']) + + cur = psql(SRC, "SELECT created_at || '|' || id FROM messages WHERE chat_id = %s ORDER BY created_at, id OFFSET 54999 LIMIT 1" % BIG) + cursor_ts, cursor_id = cur.split('|') + qs = queries(cursor_ts, cursor_id) + + log('3/6 index / query comparison') + result['read'] = {} + result['write'] = {} + for name in INDEX_CONFIGS: + apply_config(name) + size = psql(SRC, "SELECT coalesce(sum(pg_relation_size(indexrelid)), 0) FROM pg_index WHERE indrelid = 'messages'::regclass AND NOT indisprimary") + result['read'][name] = {q: explain_many(sql, a.repeat) for q, sql in qs.items()} + result['write'][name] = write_cost(max(4, a.repeat // 2)) + result['write'][name]['secondary_index_mb'] = round(int(size) / 1048576, 2) + log(' %-16s deep_keyset=%.3fms offset=%.3fms small=%.3fms insert20k=%.0fms' % ( + name, result['read'][name]['big_deep_keyset']['exec_ms_median'], + result['read'][name]['big_deep_offset']['exec_ms_median'], + result['read'][name]['small_first_page']['exec_ms_median'], + result['write'][name]['insert_20k_ms_median'])) + apply_config(PRODUCTION) + (OUT / 'explain_production_deep_keyset.txt').write_text( + psql(SRC, 'EXPLAIN (ANALYZE, BUFFERS) ' + qs['big_deep_keyset']) + '\n', encoding='utf-8') + (OUT / 'explain_pk_only_deep_keyset.txt').write_text('see results.json -> read.pk_only\n', encoding='utf-8') + + log('4/6 backup (pg_dump -Fc)') + _, dump_s = timed(lambda: compose('exec', '-T', SRC, 'pg_dump', '-U', 'messenger', '-d', 'messenger', '-Fc', '-f', '/tmp/messenger.dump')) + dump_bytes = int(compose('exec', '-T', SRC, 'stat', '-c', '%s', '/tmp/messenger.dump').stdout.strip()) + backup_at = psql(SRC, 'SELECT now()') + psql(SRC, "INSERT INTO messages (id, chat_id, sender_id, text) VALUES (md5('after-backup')::uuid, %s, md5('user8')::uuid, 'written after backup')" % BIG) + (ROOT / 'backups').mkdir(exist_ok=True) + _, copy_s = timed(lambda: compose('cp', SRC + ':/tmp/messenger.dump', 'backups/messenger.dump')) + + log('5/6 restore into NEW storage (fresh volume)') + fresh(DST) + _, copy2_s = timed(lambda: compose('cp', 'backups/messenger.dump', DST + ':/tmp/messenger.dump')) + _, restore_s = timed(lambda: compose('exec', '-T', DST, 'pg_restore', '-U', 'messenger', '-d', 'messenger', + '--no-owner', '--exit-on-error', '-j', '4', '/tmp/messenger.dump')) + _, analyze_s = timed(lambda: psql(DST, 'ANALYZE')) + result['backup_restore'] = { + 'backup_taken_at': backup_at, + 'dump_seconds': round(dump_s, 2), + 'dump_mb': round(dump_bytes / 1048576, 1), + 'copy_out_seconds': round(copy_s, 2), + 'copy_in_seconds': round(copy2_s, 2), + 'pg_restore_seconds': round(restore_s, 2), + 'analyze_seconds': round(analyze_s, 2), + 'total_recovery_seconds': round(copy2_s + restore_s + analyze_s, 2), + } + log(' %s' % result['backup_restore']) + + log('6/6 verification source vs restored') + dst_v = verify(DST) + after_src = psql(SRC, "SELECT count(*) FROM messages WHERE id = md5('after-backup')::uuid") + after_dst = psql(DST, "SELECT count(*) FROM messages WHERE id = md5('after-backup')::uuid") + psql(SRC, "DELETE FROM messages WHERE id = md5('after-backup')::uuid") + src_v_at_backup = verify(SRC) + checks = {k: {'source': src_v_at_backup[k], 'restored': dst_v[k], 'match': src_v_at_backup[k] == dst_v[k]} for k in VERIFY} + restored_plan = psql(DST, 'EXPLAIN ' + qs['big_deep_keyset']).splitlines()[0:3] + result['verification'] = { + 'checks': checks, + 'all_match': all(c['match'] for c in checks.values()), + 'post_backup_write_in_source': after_src == '1', + 'post_backup_write_in_restored': after_dst == '1', + 'restored_history_plan': restored_plan, + } + result['finished_at'] = dt.datetime.now(dt.timezone.utc).isoformat(timespec='seconds') + (OUT / 'results.json').write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + for k, c in checks.items(): + log(' %-24s %s' % (k, 'OK' if c['match'] else 'MISMATCH')) + log('post-backup write: source=%s restored=%s (expected: lost in restore)' % (after_src, after_dst)) + log('saved evidence/track2/results.json') + compose('stop', SRC, DST) + sys.exit(0 if result['verification']['all_match'] else 1) + + +if __name__ == '__main__': + main() diff --git a/scripts/track2/seed.sql b/scripts/track2/seed.sql new file mode 100644 index 00000000..8a42c325 --- /dev/null +++ b/scripts/track2/seed.sql @@ -0,0 +1,35 @@ +\set ON_ERROR_STOP on +SELECT setseed(0.42); + +INSERT INTO users (id, username, display_name, password_hash, created_at) +SELECT md5('user' || g)::uuid, 'r_' || g, 'Research user ' || g, 'not-a-real-hash', + timestamptz '2025-12-01 00:00:00Z' + FROM generate_series(1, 2000) g; + +CREATE TEMP TABLE plan AS +SELECT c, + CASE WHEN c = 1 THEN 50 WHEN c <= 21 THEN 10 ELSE 2 + c % 2 END AS k, + CASE WHEN c = 1 THEN 60000 WHEN c <= 21 THEN 1500 ELSE 1 + (c * 37) % 19 END AS n + FROM generate_series(1, 2021) c; + +INSERT INTO chats (id, title, created_by, created_at) +SELECT md5('chat' || c)::uuid, 'Research chat ' || c, md5('user' || ((c * 7) % 2000 + 1))::uuid, + timestamptz '2026-01-01 00:00:00Z' + c * interval '1 second' + FROM plan; + +INSERT INTO chat_members (chat_id, user_id, position) +SELECT md5('chat' || p.c)::uuid, md5('user' || ((p.c * 7 + pos) % 2000 + 1))::uuid, pos + FROM plan p, generate_series(0, p.k - 1) pos; + +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT md5('msg' || p.c || ':' || j)::uuid, + md5('chat' || p.c)::uuid, + md5('user' || ((p.c * 7 + j % p.k) % 2000 + 1))::uuid, + left(repeat(md5(p.c || ':' || j), 1 + floor(power(random(), 4) * 62)::int), 4000), + date_trunc('milliseconds', + timestamptz '2026-01-02 00:00:00Z' + + (j::float8 / p.n) * interval '180 days' + + floor(random() * 1000) * interval '1 millisecond') + FROM plan p, generate_series(1, p.n) j; + +ANALYZE; diff --git a/src/cmd/server/main.go b/src/cmd/server/main.go new file mode 100644 index 00000000..07556ff1 --- /dev/null +++ b/src/cmd/server/main.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "mtusi-messenger/internal/api" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/cache" + "mtusi-messenger/internal/db" + "mtusi-messenger/internal/store" +) + +func getenv(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func getint(key string, def int) int { + v := os.Getenv(key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + fmt.Fprintf(os.Stderr, "invalid %s\n", key) + os.Exit(2) + } + return n +} + +func secret(key string) (string, error) { + if path := os.Getenv(key + "_FILE"); path != "" { + b, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s_FILE: %w", key, err) + } + return strings.TrimSpace(string(b)), nil + } + if v := os.Getenv(key); v != "" { + return v, nil + } + return "", fmt.Errorf("%s or %s_FILE is required", key, key) +} + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + if err := run(log); err != nil { + log.Error("fatal", "error", err.Error()) + os.Exit(1) + } +} + +func openDB(ctx context.Context) (*store.Store, func(), error) { + dsn, err := secret("DATABASE_URL") + if err != nil { + return nil, nil, err + } + pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: int32(getint("DB_MAX_CONNS", 10))}) + if err != nil { + return nil, nil, errors.New("database config is invalid") + } + return store.New(pool), pool.Close, nil +} + +func run(log *slog.Logger) error { + ctx := context.Background() + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + dsn, err := secret("DATABASE_URL") + if err != nil { + return err + } + pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: 2}) + if err != nil { + return errors.New("database config is invalid") + } + defer pool.Close() + mctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + return db.Migrate(mctx, pool, log) + } + + st, closeDB, err := openDB(ctx) + if err != nil { + return err + } + defer closeDB() + + ttl := time.Duration(getint("SESSION_TTL_SECONDS", 3600)) * time.Second + cacheMaxTTL := time.Duration(getint("SESSION_CACHE_MAX_TTL_SECONDS", 60)) * time.Second + + var sessCache auth.SessionCache + switch mode := getenv("SESSION_STORE", "postgres"); mode { + case "postgres": + case "valkey", "redis": + url, err := secret("CACHE_URL") + if err != nil { + return err + } + vk, err := cache.NewValkey(url) + if err != nil { + return errors.New("cache config is invalid") + } + defer vk.Close() + sessCache = vk + default: + return fmt.Errorf("unknown SESSION_STORE %q (postgres|valkey|redis)", mode) + } + + hasher := auth.NewHasher(getint("PASSWORD_HASH_CONCURRENCY", 4)) + sessions := auth.NewSessions(st, sessCache, ttl, cacheMaxTTL, log) + + ready := &atomic.Bool{} + ready.Store(true) + + host := getenv("APP_HOST", "0.0.0.0") + port := getenv("APP_PORT", "8080") + addr := host + ":" + port + srv := api.NewServer(st, sessions, hasher, log, ready) + httpSrv := &http.Server{ + Addr: addr, + Handler: srv.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, + } + + bgCtx, stopBg := context.WithCancel(ctx) + defer stopBg() + go func() { + t := time.NewTicker(10 * time.Minute) + defer t.Stop() + for { + select { + case <-bgCtx.Done(): + return + case <-t.C: + c, cancel := context.WithTimeout(bgCtx, 10*time.Second) + if n, err := st.PurgeSessions(c, 24*time.Hour); err == nil && n > 0 { + log.Info("expired sessions purged", "count", n) + } + cancel() + } + } + }() + + errCh := make(chan error, 1) + go func() { errCh <- httpSrv.ListenAndServe() }() + log.Info("server started", "addr", addr, "pid", os.Getpid(), + "session_store", getenv("SESSION_STORE", "postgres"), "session_ttl_s", int(ttl.Seconds())) + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + + select { + case err := <-errCh: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + case sig := <-stop: + log.Info("shutdown signal received", "signal", sig.String()) + ready.Store(false) + sctx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + if err := httpSrv.Shutdown(sctx); err != nil { + log.Warn("graceful shutdown exceeded budget, closing", "error", err.Error()) + _ = httpSrv.Close() + } + log.Info("server stopped cleanly") + } + return nil +} diff --git a/src/go.mod b/src/go.mod new file mode 100644 index 00000000..e4a5be61 --- /dev/null +++ b/src/go.mod @@ -0,0 +1,21 @@ +module mtusi-messenger + +go 1.23 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.7.2 + github.com/redis/go-redis/v9 v9.7.0 + golang.org/x/crypto v0.31.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect +) diff --git a/src/go.sum b/src/go.sum new file mode 100644 index 00000000..610cf610 --- /dev/null +++ b/src/go.sum @@ -0,0 +1,42 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/internal/api/handlers.go b/src/internal/api/handlers.go new file mode 100644 index 00000000..b5477fac --- /dev/null +++ b/src/internal/api/handlers.go @@ -0,0 +1,377 @@ +package api + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "strings" + "time" + "unicode/utf8" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/store" +) + +func (s *Server) storeError(w http.ResponseWriter, r *http.Request, err error) { + id := requestID(r) + if store.IsUnavailable(err) || errors.Is(err, auth.ErrCacheUnavailable) { + s.log.Warn("dependency unavailable", "request_id", id, "error", err.Error()) + apperr.Write(w, id, apperr.Unavailable("storage temporarily unavailable")) + return + } + s.log.Error("unexpected storage error", "request_id", id, "error", err.Error()) + apperr.Write(w, id, apperr.Internal("internal error")) +} + +func (s *Server) healthLive(w http.ResponseWriter, r *http.Request) { + writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"}) +} + +func (s *Server) healthReady(w http.ResponseWriter, r *http.Request) { + if !s.ready.Load() { + apperr.Write(w, requestID(r), apperr.Unavailable("service is shutting down")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), time.Second) + defer cancel() + if err := s.store.Ping(ctx); err != nil { + apperr.Write(w, requestID(r), apperr.Unavailable("database is not reachable")) + return + } + writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"}) +} + +type createUserReq struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +func validPassword(p string) bool { + n := utf8.RuneCountInString(p) + return utf8.ValidString(p) && n >= 12 && n <= 128 +} + +func (s *Server) usersRoute(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + id := requestID(r) + var req createUserReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + apperr.Write(w, id, aerr) + return + } + if !usernameRe.MatchString(req.Username) { + apperr.Write(w, id, apperr.InvalidRequest("username must match ^[a-z0-9_]{3,32}$")) + return + } + if !validText(req.DisplayName, 100) { + apperr.Write(w, id, apperr.InvalidRequest("display_name must be 1..100 non-space code points")) + return + } + if !validPassword(req.Password) { + apperr.Write(w, id, apperr.InvalidRequest("password must be 12..128 code points")) + return + } + hash, err := s.hasher.Hash(r.Context(), req.Password) + if err != nil { + s.storeError(w, r, err) + return + } + u, err := s.store.CreateUser(r.Context(), req.Username, req.DisplayName, hash) + if errors.Is(err, store.ErrUsernameTaken) { + apperr.Write(w, id, apperr.Conflict("username is already taken")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, u) +} + +func (s *Server) me(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + writeJSON(w, r, http.StatusOK, currentUser(r)) +} + +const basicChallenge = `Basic realm="messenger", charset="UTF-8"` + +func unauthorizedBasic(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", basicChallenge) + apperr.Write(w, requestID(r), apperr.Unauthorized("invalid username or password")) +} + +func parseBasic(r *http.Request) (string, string, bool) { + scheme, value, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Basic") { + return "", "", false + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value)) + if err != nil || !utf8.Valid(raw) { + return "", "", false + } + user, pass, ok := strings.Cut(string(raw), ":") + if !ok || !usernameRe.MatchString(user) || !validPassword(pass) { + return "", "", false + } + return user, pass, true +} + +func (s *Server) createSession(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + username, password, ok := parseBasic(r) + if !ok { + unauthorizedBasic(w, r) + return + } + user, hash, err := s.store.Credentials(r.Context(), username) + if err != nil && !errors.Is(err, store.ErrNotFound) { + s.storeError(w, r, err) + return + } + + valid, err := s.hasher.Verify(r.Context(), password, hash) + if err != nil { + s.storeError(w, r, err) + return + } + if !valid { + unauthorizedBasic(w, r) + return + } + token, expires, err := s.sessions.Create(r.Context(), user) + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, model.Session{ + Token: token, TokenType: "Bearer", ExpiresAt: model.NewTime(expires), User: *user, + }) +} + +func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + methodNotAllowed(w, r) + return + } + err := s.sessions.Revoke(r.Context(), currentToken(r)) + if errors.Is(err, auth.ErrInvalidSession) { + unauthorizedBearer(w, r) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + w.Header().Set("X-Request-Id", requestID(r)) + w.WriteHeader(http.StatusNoContent) +} + +type createChatReq struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +func (s *Server) chatsRoute(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + s.createChat(w, r) + case http.MethodGet: + s.listChats(w, r) + default: + methodNotAllowed(w, r) + } +} + +func (s *Server) createChat(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + var req createChatReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + apperr.Write(w, id, aerr) + return + } + if !validText(req.Title, 100) { + apperr.Write(w, id, apperr.InvalidRequest("title must be 1..100 non-space code points")) + return + } + if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 { + apperr.Write(w, id, apperr.InvalidRequest("member_ids must contain 1..100 items")) + return + } + seen := make(map[string]bool, len(req.MemberIDs)) + for _, raw := range req.MemberIDs { + uid, err := parseUUID(raw) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("member_ids must be UUID strings")) + return + } + if seen[uid] { + apperr.Write(w, id, apperr.InvalidRequest("duplicate member_id")) + return + } + seen[uid] = true + } + caller := currentUser(r) + if !seen[caller.ID] { + apperr.Write(w, id, apperr.InvalidRequest("creator must be included in member_ids")) + return + } + + chat, err := s.store.CreateChat(r.Context(), caller.ID, req.Title, req.MemberIDs) + if errors.Is(err, store.ErrUnknownMember) { + apperr.Write(w, id, apperr.InvalidRequest("unknown member_id")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, chat) +} + +func (s *Server) listChats(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + caller := currentUser(r) + limit, cur, aerr := pagingParams(r, caller.ID) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + items, more, err := s.store.ListChats(r.Context(), caller.ID, limit, cur) + if err != nil { + s.storeError(w, r, err) + return + } + page := model.Page[*model.Chat]{Items: items} + if more && len(items) > 0 { + last := items[len(items)-1] + encoded := store.EncodeCursor(store.Cursor{Scope: caller.ID, TS: last.CreatedAt.Time, ID: last.ID}) + page.NextCursor = &encoded + } + writeJSON(w, r, http.StatusOK, page) +} + +func (s *Server) chatRoute(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return + } + chat, err := s.store.ChatForUser(r.Context(), chatID, currentUser(r).ID) + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusOK, chat) +} + +type createMessageReq struct { + Text string `json:"text"` +} + +func (s *Server) messagesRoute(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + s.createMessage(w, r) + case http.MethodGet: + s.listMessages(w, r) + default: + methodNotAllowed(w, r) + } +} + +func (s *Server) chatAccess(w http.ResponseWriter, r *http.Request) (string, bool) { + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return "", false + } + ok, err := s.store.IsMember(r.Context(), chatID, currentUser(r).ID) + if err != nil { + s.storeError(w, r, err) + return "", false + } + if !ok { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return "", false + } + return chatID, true +} + +func (s *Server) createMessage(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + chatID, ok := s.chatAccess(w, r) + if !ok { + return + } + var req createMessageReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + apperr.Write(w, id, aerr) + return + } + if !validText(req.Text, 4000) { + apperr.Write(w, id, apperr.InvalidRequest("text must be 1..4000 non-space code points")) + return + } + msg, err := s.store.AddMessage(r.Context(), chatID, currentUser(r).ID, req.Text) + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, msg) +} + +func (s *Server) listMessages(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + chatID, ok := s.chatAccess(w, r) + if !ok { + return + } + limit, cur, aerr := pagingParams(r, chatID) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + items, more, err := s.store.ListMessages(r.Context(), chatID, limit, cur) + if err != nil { + s.storeError(w, r, err) + return + } + page := model.Page[*model.Message]{Items: items} + if more && len(items) > 0 { + last := items[len(items)-1] + encoded := store.EncodeCursor(store.Cursor{Scope: chatID, TS: last.CreatedAt.Time, ID: last.ID}) + page.NextCursor = &encoded + } + writeJSON(w, r, http.StatusOK, page) +} + +func (s *Server) notFound(w http.ResponseWriter, r *http.Request) { + apperr.Write(w, requestID(r), apperr.NotFound("route not found")) +} diff --git a/src/internal/api/helpers.go b/src/internal/api/helpers.go new file mode 100644 index 00000000..c8635562 --- /dev/null +++ b/src/internal/api/helpers.go @@ -0,0 +1,94 @@ +package api + +import ( + "encoding/json" + "errors" + "mime" + "net/http" + "regexp" + "strconv" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/store" +) + +const maxBodyBytes = 1 << 20 + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func parseUUID(s string) (string, error) { + u, err := uuid.Parse(s) + if err != nil || u.String() != s { + return "", errors.New("invalid uuid") + } + return u.String(), nil +} + +func runeLen(s string) int { return utf8.RuneCountInString(s) } + +func validText(s string, max int) bool { + n := runeLen(s) + return n >= 1 && n <= max && strings.TrimSpace(s) != "" +} + +func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) *apperr.AppError { + media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + if media != "application/json" { + return apperr.UnsupportedMedia("Content-Type must be application/json") + } + r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return apperr.PayloadTooLarge("request body too large") + } + return apperr.InvalidRequest("malformed JSON body") + } + if dec.More() { + return apperr.InvalidRequest("multiple JSON documents in body") + } + return nil +} + +func pagingParams(r *http.Request, scope string) (int, *store.Cursor, *apperr.AppError) { + limit := 50 + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 100 { + return 0, nil, apperr.InvalidRequest("limit must be an integer in 1..100") + } + limit = n + } + var cur *store.Cursor + if v := r.URL.Query().Get("cursor"); v != "" { + c, err := store.DecodeCursor(v) + if err == nil { + _, err = parseUUID(c.ID) + } + if err != nil { + return 0, nil, apperr.InvalidRequest("invalid cursor") + } + if c.Scope != scope { + return 0, nil, apperr.InvalidRequest("cursor does not belong to this resource") + } + cur = &c + } + return limit, cur, nil +} + +func writeJSON(w http.ResponseWriter, r *http.Request, status int, payload any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Request-Id", requestID(r)) + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +func methodNotAllowed(w http.ResponseWriter, r *http.Request) { + apperr.Write(w, requestID(r), apperr.New(http.StatusMethodNotAllowed, apperr.CodeInvalidRequest, "method not allowed")) +} diff --git a/src/internal/api/middleware.go b/src/internal/api/middleware.go new file mode 100644 index 00000000..9a568fa3 --- /dev/null +++ b/src/internal/api/middleware.go @@ -0,0 +1,138 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/model" +) + +type ctxKey int + +const ( + ctxRequestID ctxKey = iota + ctxUser + ctxToken +) + +func requestIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get("X-Request-Id") + if id == "" { + id = uuid.NewString() + } + w.Header().Set("X-Request-Id", id) + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxRequestID, id))) + }) +} + +func requestID(r *http.Request) string { + if id, ok := r.Context().Value(ctxRequestID).(string); ok && id != "" { + return id + } + return "unknown" +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (s *statusWriter) WriteHeader(code int) { + s.status = code + s.ResponseWriter.WriteHeader(code) +} + +func loggingMiddleware(log *slog.Logger, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + sw := &statusWriter{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(sw, r) + + log.Info("http", + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "dur_ms", time.Since(start).Milliseconds(), + "request_id", requestID(r), + ) + }) +} + +func recoverMiddleware(log *slog.Logger, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec != nil { + log.Error("panic recovered", "path", r.URL.Path, "request_id", requestID(r)) + apperr.Write(w, requestID(r), apperr.Internal("internal error")) + } + }() + next.ServeHTTP(w, r) + }) +} + +func timeoutMiddleware(d time.Duration, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), d) + defer cancel() + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +const bearerChallenge = `Bearer realm="messenger"` + +func unauthorizedBearer(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", bearerChallenge) + apperr.Write(w, requestID(r), apperr.Unauthorized("valid Bearer session token is required")) +} + +func bearerToken(r *http.Request) (string, bool) { + scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Bearer") { + return "", false + } + token = strings.TrimSpace(token) + if token == "" || len(token) > 256 { + return "", false + } + return token, true +} + +func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, ok := bearerToken(r) + if !ok { + unauthorizedBearer(w, r) + return + } + user, err := s.sessions.Resolve(r.Context(), token) + if errors.Is(err, auth.ErrInvalidSession) { + unauthorizedBearer(w, r) + return + } + if err != nil { + + s.storeError(w, r, err) + return + } + ctx := context.WithValue(r.Context(), ctxUser, user) + ctx = context.WithValue(ctx, ctxToken, token) + next.ServeHTTP(w, r.WithContext(ctx)) + } +} + +func currentUser(r *http.Request) *model.User { + return r.Context().Value(ctxUser).(*model.User) +} + +func currentToken(r *http.Request) string { + return r.Context().Value(ctxToken).(string) +} diff --git a/src/internal/api/router.go b/src/internal/api/router.go new file mode 100644 index 00000000..a0dbec99 --- /dev/null +++ b/src/internal/api/router.go @@ -0,0 +1,45 @@ +package api + +import ( + "log/slog" + "net/http" + "sync/atomic" + "time" + + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/store" +) + +type Server struct { + store *store.Store + sessions *auth.Sessions + hasher *auth.Hasher + log *slog.Logger + ready *atomic.Bool + timeout time.Duration +} + +func NewServer(st *store.Store, sessions *auth.Sessions, hasher *auth.Hasher, log *slog.Logger, ready *atomic.Bool) *Server { + return &Server{store: st, sessions: sessions, hasher: hasher, log: log, ready: ready, timeout: 8 * time.Second} +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/health/live", s.healthLive) + mux.HandleFunc("/health/ready", s.healthReady) + mux.HandleFunc("/api/v1/users", s.usersRoute) + mux.HandleFunc("/api/v1/auth/sessions", s.createSession) + mux.HandleFunc("/api/v1/auth/sessions/current", s.auth(s.revokeSession)) + mux.HandleFunc("/api/v1/users/me", s.auth(s.me)) + mux.HandleFunc("/api/v1/chats", s.auth(s.chatsRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}", s.auth(s.chatRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}/messages", s.auth(s.messagesRoute)) + mux.HandleFunc("/", s.notFound) + + var h http.Handler = mux + h = timeoutMiddleware(s.timeout, h) + h = recoverMiddleware(s.log, h) + h = loggingMiddleware(s.log, h) + h = requestIDMiddleware(h) + return h +} diff --git a/src/internal/apperr/errors.go b/src/internal/apperr/errors.go new file mode 100644 index 00000000..0c6d7c5e --- /dev/null +++ b/src/internal/apperr/errors.go @@ -0,0 +1,64 @@ +package apperr + +import ( + "encoding/json" + "net/http" +) + +type Code string + +const ( + CodeInvalidRequest Code = "invalid_request" + CodeUnauthorized Code = "unauthorized" + CodeNotFound Code = "not_found" + CodeConflict Code = "conflict" + CodePayloadTooLarge Code = "payload_too_large" + CodeUnsupportedMedia Code = "unsupported_media_type" + CodeRateLimited Code = "rate_limited" + CodeUnavailable Code = "unavailable" + CodeInternal Code = "internal_error" +) + +type Body struct { + Code Code `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type Envelope struct { + Error Body `json:"error"` +} + +type AppError struct { + Status int + Code Code + Msg string +} + +func (e *AppError) Error() string { return e.Msg } + +func New(status int, code Code, msg string) *AppError { + return &AppError{Status: status, Code: code, Msg: msg} +} + +func InvalidRequest(msg string) *AppError { return New(http.StatusBadRequest, CodeInvalidRequest, msg) } +func Unauthorized(msg string) *AppError { return New(http.StatusUnauthorized, CodeUnauthorized, msg) } +func NotFound(msg string) *AppError { return New(http.StatusNotFound, CodeNotFound, msg) } +func Conflict(msg string) *AppError { return New(http.StatusConflict, CodeConflict, msg) } +func PayloadTooLarge(msg string) *AppError { + return New(http.StatusRequestEntityTooLarge, CodePayloadTooLarge, msg) +} +func UnsupportedMedia(msg string) *AppError { + return New(http.StatusUnsupportedMediaType, CodeUnsupportedMedia, msg) +} +func Unavailable(msg string) *AppError { + return New(http.StatusServiceUnavailable, CodeUnavailable, msg) +} +func Internal(msg string) *AppError { return New(http.StatusInternalServerError, CodeInternal, msg) } + +func Write(w http.ResponseWriter, requestID string, err *AppError) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Request-Id", requestID) + w.WriteHeader(err.Status) + _ = json.NewEncoder(w).Encode(Envelope{Error: Body{Code: err.Code, Message: err.Msg, RequestID: requestID}}) +} diff --git a/src/internal/auth/password.go b/src/internal/auth/password.go new file mode 100644 index 00000000..ceaf8fe4 --- /dev/null +++ b/src/internal/auth/password.go @@ -0,0 +1,100 @@ +package auth + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + argonMemoryKiB = 19 * 1024 + argonTime = 2 + argonThreads = 1 + argonKeyLen = 32 + argonSaltLen = 16 +) + +var ErrBadHash = errors.New("malformed password hash") + +type Hasher struct { + slots chan struct{} + dummy string +} + +func NewHasher(concurrency int) *Hasher { + h := &Hasher{slots: make(chan struct{}, concurrency)} + + salt := make([]byte, argonSaltLen) + _, _ = rand.Read(salt) + h.dummy = encode(salt, argon2.IDKey([]byte("dummy-password-x"), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)) + return h +} + +func (h *Hasher) acquire(ctx context.Context) error { + select { + case h.slots <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (h *Hasher) release() { <-h.slots } + +func encode(salt, key []byte) string { + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemoryKiB, argonTime, argonThreads, + base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key)) +} + +func (h *Hasher) Hash(ctx context.Context, password string) (string, error) { + if err := h.acquire(ctx); err != nil { + return "", err + } + defer h.release() + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + return encode(salt, argon2.IDKey([]byte(password), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)), nil +} + +func (h *Hasher) Verify(ctx context.Context, password, encoded string) (bool, error) { + missing := encoded == "" + if missing { + encoded = h.dummy + } + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false, ErrBadHash + } + var version int + var mem uint32 + var t uint32 + var p uint8 + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version { + return false, ErrBadHash + } + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &t, &p); err != nil { + return false, ErrBadHash + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, ErrBadHash + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, ErrBadHash + } + if err := h.acquire(ctx); err != nil { + return false, err + } + got := argon2.IDKey([]byte(password), salt, t, mem, p, uint32(len(want))) + h.release() + return subtle.ConstantTimeCompare(got, want) == 1 && !missing, nil +} diff --git a/src/internal/auth/sessions.go b/src/internal/auth/sessions.go new file mode 100644 index 00000000..cf0a51ec --- /dev/null +++ b/src/internal/auth/sessions.go @@ -0,0 +1,137 @@ +package auth + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "log/slog" + "time" + + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/store" +) + +var ( + ErrInvalidSession = errors.New("invalid session") + ErrCacheUnavailable = errors.New("session cache unavailable") +) + +type SessionCache interface { + Get(ctx context.Context, key string) (*CachedSession, error) + Set(ctx context.Context, key string, s *CachedSession, ttl time.Duration) error + Del(ctx context.Context, key string) error +} + +type CachedSession struct { + User model.User `json:"user"` + ExpiresAt time.Time `json:"exp"` +} + +type Sessions struct { + store *store.Store + cache SessionCache + ttl time.Duration + cacheMaxTTL time.Duration + log *slog.Logger +} + +func NewSessions(st *store.Store, cache SessionCache, ttl, cacheMaxTTL time.Duration, log *slog.Logger) *Sessions { + return &Sessions{store: st, cache: cache, ttl: ttl, cacheMaxTTL: cacheMaxTTL, log: log} +} + +func NewToken() (string, []byte, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", nil, err + } + token := base64.RawURLEncoding.EncodeToString(raw) + return token, HashToken(token), nil +} + +func HashToken(token string) []byte { + sum := sha256.Sum256([]byte(token)) + return sum[:] +} + +func cacheKey(hash []byte) string { + return "sess:" + base64.RawURLEncoding.EncodeToString(hash) +} + +func (s *Sessions) Create(ctx context.Context, user *model.User) (string, time.Time, error) { + token, hash, err := NewToken() + if err != nil { + return "", time.Time{}, err + } + expires, err := s.store.CreateSession(ctx, hash, user.ID, s.ttl) + if err != nil { + return "", time.Time{}, err + } + s.cacheSet(ctx, hash, user, expires) + return token, expires, nil +} + +func (s *Sessions) Resolve(ctx context.Context, token string) (*model.User, error) { + hash := HashToken(token) + if s.cache != nil { + cs, err := s.cache.Get(ctx, cacheKey(hash)) + if err == nil && cs != nil { + if time.Now().Before(cs.ExpiresAt) { + u := cs.User + return &u, nil + } + + } else if err != nil { + s.log.Debug("session cache get failed, falling back to postgres", "error", err.Error()) + } + } + user, expires, err := s.store.SessionUser(ctx, hash) + if errors.Is(err, store.ErrNotFound) { + return nil, ErrInvalidSession + } + if err != nil { + return nil, err + } + s.cacheSet(ctx, hash, user, expires) + return user, nil +} + +func (s *Sessions) Revoke(ctx context.Context, token string) error { + hash := HashToken(token) + if s.cache != nil { + if err := s.cache.Del(ctx, cacheKey(hash)); err != nil { + return ErrCacheUnavailable + } + } + err := s.store.RevokeSession(ctx, hash) + if errors.Is(err, store.ErrNotFound) { + return ErrInvalidSession + } + if err != nil { + return err + } + if s.cache != nil { + if err := s.cache.Del(ctx, cacheKey(hash)); err != nil { + + s.log.Warn("session cache delete after revoke failed", "error", err.Error()) + } + } + return nil +} + +func (s *Sessions) cacheSet(ctx context.Context, hash []byte, user *model.User, expires time.Time) { + if s.cache == nil { + return + } + ttl := time.Until(expires) + if ttl > s.cacheMaxTTL { + ttl = s.cacheMaxTTL + } + if ttl <= 0 { + return + } + if err := s.cache.Set(ctx, cacheKey(hash), &CachedSession{User: *user, ExpiresAt: expires}, ttl); err != nil { + s.log.Debug("session cache set failed", "error", err.Error()) + } +} diff --git a/src/internal/cache/valkey.go b/src/internal/cache/valkey.go new file mode 100644 index 00000000..591d9e49 --- /dev/null +++ b/src/internal/cache/valkey.go @@ -0,0 +1,80 @@ +package cache + +import ( + "context" + "encoding/json" + "errors" + "sync/atomic" + "time" + + "github.com/redis/go-redis/v9" + + "mtusi-messenger/internal/auth" +) + +var ErrOpen = errors.New("cache circuit open") + +type Valkey struct { + rdb *redis.Client + openUntil atomic.Int64 + cooldown time.Duration +} + +func NewValkey(url string) (*Valkey, error) { + opt, err := redis.ParseURL(url) + if err != nil { + return nil, err + } + opt.DialTimeout = 500 * time.Millisecond + opt.ReadTimeout = 300 * time.Millisecond + opt.WriteTimeout = 300 * time.Millisecond + opt.PoolSize = 10 + opt.MaxRetries = 0 + return &Valkey{rdb: redis.NewClient(opt), cooldown: 2 * time.Second}, nil +} + +func (v *Valkey) Close() error { return v.rdb.Close() } + +func (v *Valkey) Ping(ctx context.Context) error { return v.guard(v.rdb.Ping(ctx).Err()) } + +func (v *Valkey) open() bool { return time.Now().UnixNano() < v.openUntil.Load() } + +func (v *Valkey) guard(err error) error { + if err != nil && !errors.Is(err, redis.Nil) { + v.openUntil.Store(time.Now().Add(v.cooldown).UnixNano()) + } + return err +} + +func (v *Valkey) Get(ctx context.Context, key string) (*auth.CachedSession, error) { + if v.open() { + return nil, ErrOpen + } + raw, err := v.rdb.Get(ctx, key).Bytes() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err := v.guard(err); err != nil { + return nil, err + } + var cs auth.CachedSession + if err := json.Unmarshal(raw, &cs); err != nil { + return nil, nil + } + return &cs, nil +} + +func (v *Valkey) Set(ctx context.Context, key string, s *auth.CachedSession, ttl time.Duration) error { + if v.open() { + return ErrOpen + } + raw, err := json.Marshal(s) + if err != nil { + return err + } + return v.guard(v.rdb.Set(ctx, key, raw, ttl).Err()) +} + +func (v *Valkey) Del(ctx context.Context, key string) error { + return v.guard(v.rdb.Del(ctx, key).Err()) +} diff --git a/src/internal/db/migrate.go b/src/internal/db/migrate.go new file mode 100644 index 00000000..96d0c779 --- /dev/null +++ b/src/internal/db/migrate.go @@ -0,0 +1,122 @@ +package db + +import ( + "context" + "embed" + "fmt" + "io/fs" + "log/slog" + "sort" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +const advisoryLockID = 20260202 + +type migration struct { + version int + name string + sql string +} + +func loadMigrations() ([]migration, error) { + entries, err := fs.ReadDir(migrationFS, "migrations") + if err != nil { + return nil, err + } + var out []migration + for _, e := range entries { + name := e.Name() + prefix, _, ok := strings.Cut(name, "_") + if !ok || !strings.HasSuffix(name, ".sql") { + return nil, fmt.Errorf("bad migration file name %q", name) + } + v, err := strconv.Atoi(prefix) + if err != nil { + return nil, fmt.Errorf("bad migration version in %q", name) + } + body, err := migrationFS.ReadFile("migrations/" + name) + if err != nil { + return nil, err + } + out = append(out, migration{version: v, name: name, sql: string(body)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version }) + for i := 1; i < len(out); i++ { + if out[i].version == out[i-1].version { + return nil, fmt.Errorf("duplicate migration version %d", out[i].version) + } + } + return out, nil +} + +func Migrate(ctx context.Context, pool *pgxpool.Pool, log *slog.Logger) error { + migs, err := loadMigrations() + if err != nil { + return err + } + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID); err != nil { + return err + } + defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID) + + if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version integer PRIMARY KEY, + name text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { + return err + } + + applied := map[int]bool{} + rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations") + if err != nil { + return err + } + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + rows.Close() + return err + } + applied[v] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + + for _, m := range migs { + if applied[m.version] { + continue + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, m.sql); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("migration %s: %w", m.name, err) + } + if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", m.version, m.name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + log.Info("migration applied", "version", m.version, "name", m.name) + } + log.Info("migrations up to date", "total", len(migs)) + return nil +} diff --git a/src/internal/db/migrations/0001_init.sql b/src/internal/db/migrations/0001_init.sql new file mode 100644 index 00000000..a500b303 --- /dev/null +++ b/src/internal/db/migrations/0001_init.sql @@ -0,0 +1,51 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE +); +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); +CREATE INDEX sessions_user_idx ON sessions (user_id); +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/src/internal/db/pool.go b/src/internal/db/pool.go new file mode 100644 index 00000000..3c818a19 --- /dev/null +++ b/src/internal/db/pool.go @@ -0,0 +1,31 @@ +package db + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +type PoolOptions struct { + MaxConns int32 +} + +func Open(ctx context.Context, url string, opt PoolOptions) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(url) + if err != nil { + return nil, err + } + cfg.MaxConns = opt.MaxConns + cfg.MinConns = 1 + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = 5 * time.Minute + cfg.HealthCheckPeriod = 15 * time.Second + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + rp := cfg.ConnConfig.RuntimeParams + rp["application_name"] = "mtusi-messenger" + rp["statement_timeout"] = "5000" + rp["lock_timeout"] = "2000" + rp["idle_in_transaction_session_timeout"] = "10000" + return pgxpool.NewWithConfig(ctx, cfg) +} diff --git a/src/internal/model/models.go b/src/internal/model/models.go new file mode 100644 index 00000000..33a8da13 --- /dev/null +++ b/src/internal/model/models.go @@ -0,0 +1,51 @@ +package model + +import "time" + +type Time struct { + time.Time +} + +func NewTime(t time.Time) Time { return Time{Time: t} } + +func (t Time) MarshalJSON() ([]byte, error) { + return []byte(`"` + t.UTC().Format("2006-01-02T15:04:05.000Z") + `"`), nil +} + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt Time `json:"created_at"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt Time `json:"created_at"` +} + +type Page[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type Health struct { + Status string `json:"status"` +} + +type Session struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt Time `json:"expires_at"` + User User `json:"user"` +} diff --git a/src/internal/store/cursor.go b/src/internal/store/cursor.go new file mode 100644 index 00000000..209405b2 --- /dev/null +++ b/src/internal/store/cursor.go @@ -0,0 +1,33 @@ +package store + +import ( + "encoding/base64" + "encoding/json" + "time" +) + +type Cursor struct { + Scope string `json:"s"` + TS time.Time `json:"t"` + ID string `json:"i"` +} + +func EncodeCursor(c Cursor) string { + b, _ := json.Marshal(c) + return base64.RawURLEncoding.EncodeToString(b) +} + +func DecodeCursor(s string) (Cursor, error) { + var c Cursor + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return c, ErrInvalidCursor + } + if err := json.Unmarshal(b, &c); err != nil { + return c, ErrInvalidCursor + } + if c.Scope == "" || c.ID == "" || c.TS.IsZero() { + return c, ErrInvalidCursor + } + return c, nil +} diff --git a/src/internal/store/postgres.go b/src/internal/store/postgres.go new file mode 100644 index 00000000..89db504b --- /dev/null +++ b/src/internal/store/postgres.go @@ -0,0 +1,306 @@ +package store + +import ( + "context" + "errors" + "net" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "mtusi-messenger/internal/model" +) + +var ( + ErrNotFound = errors.New("not found") + ErrUsernameTaken = errors.New("username taken") + ErrUnknownMember = errors.New("unknown member") + ErrInvalidCursor = errors.New("invalid cursor") +) + +type Store struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } + +func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) } + +func IsUnavailable(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || pgconn.Timeout(err) { + return true + } + var connErr *pgconn.ConnectError + if errors.As(err, &connErr) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) { + return true + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + + switch { + case pgErr.Code == "57P01", pgErr.Code == "57P03", pgErr.Code == "57014", pgErr.Code == "55P03", + len(pgErr.Code) == 5 && pgErr.Code[:2] == "08": + return true + } + } + return errors.Is(err, pgx.ErrTxClosed) || errors.Is(err, net.ErrClosed) +} + +func pgCode(err error) string { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code + } + return "" +} + +func utc(t time.Time) model.Time { return model.NewTime(t.UTC()) } + +func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) { + u := &model.User{ID: uuid.NewString(), Username: username, DisplayName: displayName} + var created time.Time + err := s.pool.QueryRow(ctx, + `INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) RETURNING created_at`, + u.ID, username, displayName, passwordHash).Scan(&created) + if pgCode(err) == "23505" { + return nil, ErrUsernameTaken + } + if err != nil { + return nil, err + } + u.CreatedAt = utc(created) + return u, nil +} + +func (s *Store) Credentials(ctx context.Context, username string) (*model.User, string, error) { + u := &model.User{} + var created time.Time + var hash string + err := s.pool.QueryRow(ctx, + `SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1`, + username).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &hash) + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", ErrNotFound + } + if err != nil { + return nil, "", err + } + u.CreatedAt = utc(created) + return u, hash, nil +} + +func (s *Store) CreateSession(ctx context.Context, tokenHash []byte, userID string, ttl time.Duration) (time.Time, error) { + var expires time.Time + err := s.pool.QueryRow(ctx, + `INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES ($1, $2, date_trunc('milliseconds', now()) + make_interval(secs => $3)) + RETURNING expires_at`, + tokenHash, userID, ttl.Seconds()).Scan(&expires) + if err != nil { + return time.Time{}, err + } + return expires.UTC(), nil +} + +func (s *Store) SessionUser(ctx context.Context, tokenHash []byte) (*model.User, time.Time, error) { + u := &model.User{} + var created, expires time.Time + err := s.pool.QueryRow(ctx, + `SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`, + tokenHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &expires) + if errors.Is(err, pgx.ErrNoRows) { + return nil, time.Time{}, ErrNotFound + } + if err != nil { + return nil, time.Time{}, err + } + u.CreatedAt = utc(created) + return u, expires.UTC(), nil +} + +func (s *Store) RevokeSession(ctx context.Context, tokenHash []byte) error { + tag, err := s.pool.Exec(ctx, + `UPDATE sessions SET revoked_at = now() + WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()`, tokenHash) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (s *Store) PurgeSessions(ctx context.Context, olderThan time.Duration) (int64, error) { + tag, err := s.pool.Exec(ctx, + `DELETE FROM sessions + WHERE expires_at < now() - make_interval(secs => $1) + OR revoked_at < now() - make_interval(secs => $1)`, olderThan.Seconds()) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Store) CreateChat(ctx context.Context, creatorID, title string, memberIDs []string) (*model.Chat, error) { + c := &model.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), memberIDs...)} + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var created time.Time + if err := tx.QueryRow(ctx, + `INSERT INTO chats (id, title, created_by) VALUES ($1, $2, $3) RETURNING created_at`, + c.ID, title, creatorID).Scan(&created); err != nil { + return err + } + c.CreatedAt = utc(created) + _, err := tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1, m.user_id, m.ord - 1 + FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord)`, + c.ID, memberIDs) + return err + }) + if pgCode(err) == "23503" { + return nil, ErrUnknownMember + } + if err != nil { + return nil, err + } + return c, nil +} + +func (s *Store) IsMember(ctx context.Context, chatID, userID string) (bool, error) { + var ok bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + return ok, err +} + +func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (*model.Chat, error) { + c := &model.Chat{} + var created time.Time + err := s.pool.QueryRow(ctx, + `SELECT c.id, c.title, c.created_at, + array_agg(m.user_id::text ORDER BY m.position) + FROM chats c JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = $2) + GROUP BY c.id`, + chatID, userID).Scan(&c.ID, &c.Title, &created, &c.MemberIDs) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + c.CreatedAt = utc(created) + return c, nil +} + +func (s *Store) ListChats(ctx context.Context, userID string, limit int, cur *Cursor) ([]*model.Chat, bool, error) { + var ts *time.Time + var cid *string + if cur != nil { + ts, cid = &cur.TS, &cur.ID + } + rows, err := s.pool.Query(ctx, + `SELECT c.id, c.title, c.created_at, + (SELECT array_agg(m.user_id::text ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) + FROM chat_members me JOIN chats c ON c.id = me.chat_id + WHERE me.user_id = $1 + AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + ORDER BY c.created_at, c.id + LIMIT $4`, + userID, ts, cid, limit+1) + if err != nil { + return nil, false, err + } + defer rows.Close() + out := make([]*model.Chat, 0, limit+1) + for rows.Next() { + c := &model.Chat{} + var created time.Time + if err := rows.Scan(&c.ID, &c.Title, &created, &c.MemberIDs); err != nil { + return nil, false, err + } + c.CreatedAt = utc(created) + out = append(out, c) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + more := len(out) > limit + if more { + out = out[:limit] + } + return out, more, nil +} + +func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string) (*model.Message, error) { + m := &model.Message{ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text} + var created time.Time + err := s.pool.QueryRow(ctx, + `INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4) RETURNING created_at`, + m.ID, chatID, senderID, text).Scan(&created) + if pgCode(err) == "23503" { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + m.CreatedAt = utc(created) + return m, nil +} + +func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cur *Cursor) ([]*model.Message, bool, error) { + var ( + rows pgx.Rows + err error + ) + if cur == nil { + rows, err = s.pool.Query(ctx, + `SELECT id, chat_id, sender_id, text, created_at FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id LIMIT $2`, chatID, limit+1) + } else { + rows, err = s.pool.Query(ctx, + `SELECT id, chat_id, sender_id, text, created_at FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3::uuid) + ORDER BY created_at, id LIMIT $4`, chatID, cur.TS, cur.ID, limit+1) + } + if err != nil { + return nil, false, err + } + defer rows.Close() + out := make([]*model.Message, 0, limit+1) + for rows.Next() { + m := &model.Message{} + var created time.Time + if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created); err != nil { + return nil, false, err + } + m.CreatedAt = utc(created) + out = append(out, m) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + more := len(out) > limit + if more { + out = out[:limit] + } + return out, more, nil +} diff --git a/student-tests/run_tests.py b/student-tests/run_tests.py new file mode 100644 index 00000000..f78303b3 --- /dev/null +++ b/student-tests/run_tests.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +RESULTS = [] + + +def compose(*args, check_rc=True): + proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8') + if check_rc and proc.returncode != 0: + raise Failure('docker compose ' + ' '.join(args[:2]) + ' failed') + return proc.stdout + + +def psql(sql): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAX', '-c', sql).strip() + + +def valkey(*cmd): + script = 'valkey-cli --no-auth-warning -a "$(cut -d" " -f2 /run/secrets/valkey_auth)" ' + ' '.join(cmd) + return compose('exec', '-T', 'valkey', 'sh', '-c', script).strip() + + +def basic(username, password): + return 'Basic ' + base64.b64encode((username + ':' + password).encode()).decode() + + +def wait_ready(c, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + time.sleep(0.5) + raise Failure('API not ready after %ds' % timeout) + + +def test(name): + def deco(fn): + fn.test_name = name + return fn + return deco + + +@test('wrong password / unknown user -> одинаковый 401 Basic') +def t_wrong_password(c, a): + alice = c.user('wp') + u = alice['registration']['username'] + r1 = c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': basic(u, 'wrong-password-123')}) + r2 = c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': basic('no_such_' + uuid.uuid4().hex[:8], 'wrong-password-123')}) + check(r1['json']['error']['message'] == r2['json']['error']['message'], 'responses reveal user existence') + c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic !!!not-base64'}) + c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Bearer ' + alice['token']}) + t1 = min(c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': basic(u, 'wrong-password-123')})['elapsed'] for _ in range(5)) + t2 = min(c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': basic('nouser_x1', 'wrong-password-123')})['elapsed'] for _ in range(5)) + check(t2 > t1 * 0.5, 'unknown user answered much faster than known one (%.3f vs %.3f s)' % (t2, t1)) + return 'known=%.0fms unknown=%.0fms' % (t1 * 1000, t2 * 1000) + + +@test('пароль: 12..128 code points, двоеточие допустимо') +def t_password_rules(c, a): + name = 'u_pw_' + uuid.uuid4().hex[:10] + c.call('POST', '/api/v1/users', 400, body={'username': name, 'display_name': 'x', 'password': 'short'}) + c.call('POST', '/api/v1/users', 400, body={'username': name, 'display_name': 'x', 'password': 'x' * 129}) + pw = 'пароль:с:двоеточием🙂' + c.call('POST', '/api/v1/users', 201, body={'username': name, 'display_name': 'x', 'password': pw}, schema='User') + c.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': basic(name, pw)}, schema='Session') + + +@test('logout: 204, затем 401 на запрос и повторный logout') +def t_logout(c, a): + alice = c.user('lo') + second = c.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': basic(alice['registration']['username'], alice['registration']['password'])}, schema='Session')['json'] + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + c.call('GET', '/api/v1/users/me', 401, actor=alice) + c.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=alice) + c.call('GET', '/api/v1/users/me', 200, headers={'Authorization': 'Bearer ' + second['token']}, schema='User') + + +@test('токен непрозрачный, в БД только SHA-256, пароль — Argon2id') +def t_storage_format(c, a): + alice = c.user('st') + tok = alice['token'] + check(len(tok) >= 43 and alice['id'] not in tok and alice['registration']['username'] not in tok, 'token is not opaque') + check(psql("SELECT count(*) FROM sessions WHERE encode(token_hash,'escape') LIKE '%%%s%%'" % tok[:20]) == '0', 'raw token stored') + h = psql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id']) + check(h.startswith('$argon2id$v=19$m=19456,t=2,p=1$'), 'password is not Argon2id') + check(alice['registration']['password'] not in h, 'plaintext password stored') + + +@test('expiry при SESSION_TTL_SECONDS=2 (api-short-ttl :8081)') +def t_expiry(c, a): + compose('--profile', 'ttl', 'up', '-d', 'api-short-ttl') + short = Client(a.short_ttl_url, timeout=10) + wait_ready(short) + u = short.user('ttl') + sess_exp = psql("SELECT extract(epoch FROM s.expires_at - s.created_at) FROM sessions s JOIN users u ON u.id=s.user_id WHERE u.id='%s'" % u['id']) + check(abs(float(sess_exp) - 2) < 0.01, 'TTL is not 2 seconds: ' + sess_exp) + short.call('GET', '/api/v1/users/me', 200, actor=u, schema='User') + time.sleep(2.5) + short.call('GET', '/api/v1/users/me', 401, actor=u) + c.call('GET', '/api/v1/users/me', 401, actor=u) + compose('--profile', 'ttl', 'stop', 'api-short-ttl') + + +@test('гонка регистрации одного username: ровно один 201') +def t_register_race(c, a): + name = 'u_race_' + uuid.uuid4().hex[:10] + codes = [] + lock = threading.Lock() + barrier = threading.Barrier(20) + + def worker(): + cl = Client(a.base_url, timeout=30) + barrier.wait() + r = cl.call('POST', '/api/v1/users', [201, 409], body={'username': name, 'display_name': 'race', 'password': 'Race-password-123'}) + with lock: + codes.append(r['status']) + threads = [threading.Thread(target=worker) for _ in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + check(codes.count(201) == 1 and codes.count(409) == 19, 'unexpected codes: %s' % sorted(codes)) + check(psql("SELECT count(*) FROM users WHERE username='%s'" % name) == '1', 'duplicate user rows') + return '201x1, 409x19' + + +@test('rollback составной операции создания чата') +def t_chat_rollback(c, a): + alice = c.user('rb') + title = 'rollback-' + uuid.uuid4().hex + before = int(psql("SELECT xact_rollback FROM pg_stat_database WHERE datname='messenger'")) + c.call('POST', '/api/v1/chats', 400, actor=alice, body={'title': title, 'member_ids': [alice['id'], str(uuid.uuid4())]}) + check(psql("SELECT count(*) FROM chats WHERE title='%s'" % title) == '0', 'chat row survived failed transaction') + check(psql("SELECT count(*) FROM chat_members m JOIN chats ch ON ch.id=m.chat_id WHERE ch.title='%s'" % title) == '0', 'orphan members') + deadline = time.monotonic() + 15 + while True: + after = int(psql("SELECT xact_rollback FROM pg_stat_database WHERE datname='messenger'")) + if after > before or time.monotonic() > deadline: + break + time.sleep(0.5) + check(after > before, 'no rollback observed in pg_stat_database') + listed = c.call('GET', '/api/v1/chats?limit=100', actor=alice, schema='ChatPage')['json'] + check(all(x['title'] != title for x in listed['items']), 'rolled back chat is visible') + return 'xact_rollback %d -> %d' % (before, after) + + +@test('параметризованные запросы: SQL-символы сохраняются как текст') +def t_sql_text(c, a): + alice, bob, eve, chat = c.bootstrap() + payload = "'); DROP TABLE messages; -- \\x00? $1 %s" + m = c.message(alice, chat, payload)['json'] + check(m['text'] == payload, 'text changed') + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json'] + check(page['items'][-1]['text'] == payload, 'text changed on read') + c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?cursor=' + quote("' OR 1=1 --"), 400, actor=bob) + + +@test('пагинация 250 сообщений: без потерь и повторов, курсор чужого чата -> 400') +def t_pagination(c, a): + alice, bob, eve, chat = c.bootstrap() + sent = [c.message(alice, chat, 'm%03d' % i)['json']['id'] for i in range(250)] + seen, cursor, pages = [], None, 0 + while True: + p = '/api/v1/chats/' + chat['id'] + '/messages?limit=37' + ('&cursor=' + quote(cursor, safe='') if cursor else '') + page = c.call('GET', p, actor=bob, schema='MessagePage')['json'] + seen += [m['id'] for m in page['items']] + pages += 1 + cursor = page['next_cursor'] + if not cursor: + break + last_cursor = cursor + check(seen == sent, 'history order/content mismatch') + other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'другой', 'member_ids': [alice['id']]}, schema='Chat')['json'] + c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(last_cursor, safe=''), 400, actor=alice) + return '%d pages' % pages + + +@test('Valkey реально кэширует сессии (TTL <= 60s), FLUSHALL не разлогинивает') +def t_cache_used(c, a): + alice = c.user('cache') + c.call('GET', '/api/v1/users/me', actor=alice, schema='User') + keys = valkey('--scan', '--pattern', "'sess:*'").split() + check(keys, 'no session keys in valkey') + ttl = int(valkey('TTL', keys[0])) + check(0 < ttl <= 60, 'cache TTL not bounded: %d' % ttl) + check(alice['token'] not in ' '.join(keys), 'raw token used as cache key') + valkey('FLUSHALL') + c.call('GET', '/api/v1/users/me', 200, actor=alice, schema='User') + check(valkey('DBSIZE') != '0', 'cache was not repopulated after miss') + return 'keys=%d ttl=%ds' % (len(keys), ttl) + + +@test('отказ Valkey: API обходит кэш, logout -> 503 (не оставляет отозванный токен в кэше)') +def t_valkey_down(c, a): + alice = c.user('vk') + chat = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'vk', 'member_ids': [alice['id']]}, schema='Chat')['json'] + compose('stop', 'valkey') + try: + c.call('GET', '/health/ready', 200, schema='Health') + t0 = time.monotonic() + c.call('GET', '/api/v1/users/me', 200, actor=alice, schema='User') + c.message(alice, chat, 'без кэша') + fresh = c.user('vk2') + c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Bearer ' + 'x' * 43}) + c.call('GET', '/api/v1/users/me', 401) + c.call('DELETE', '/api/v1/auth/sessions/current', 503, actor=alice) + elapsed = time.monotonic() - t0 + finally: + compose('start', 'valkey') + deadline = time.monotonic() + 30 + while True: + r = c.call('DELETE', '/api/v1/auth/sessions/current', [204, 503], actor=alice) + if r['status'] == 204: + break + check(time.monotonic() < deadline, 'logout did not recover after valkey start') + time.sleep(0.5) + c.call('GET', '/api/v1/users/me', 401, actor=alice) + c.call('GET', '/api/v1/users/me', 200, actor=fresh, schema='User') + return 'degraded requests took %.2fs total' % elapsed + + +@test('отказ PostgreSQL: ready -> 503, бизнес-запросы 503, аноним/неизвестный токен не проходят') +def t_postgres_down(c, a): + alice, bob, eve, chat = c.bootstrap() + msg = c.message(alice, chat, 'до отказа БД')['json'] + compose('stop', 'postgres') + try: + c.call('GET', '/health/live', 200, schema='Health') + c.call('GET', '/health/ready', 503) + c.call('GET', '/api/v1/chats', 503, actor=alice) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 503, actor=alice, body={'text': 'во время отказа'}) + c.call('GET', '/api/v1/users/me', 401) + c.call('GET', '/api/v1/users/me', 503, headers={'Authorization': 'Bearer ' + 'y' * 43}) + c.call('POST', '/api/v1/users', 503, body={'username': 'u_' + uuid.uuid4().hex[:10], 'display_name': 'x', 'password': 'Outage-password-1'}) + finally: + compose('start', 'postgres') + wait_ready(c) + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json'] + check(page['items'] == [msg], 'data changed across database outage') + + +TESTS = [t_wrong_password, t_password_rules, t_logout, t_storage_format, t_expiry, t_register_race, + t_chat_rollback, t_sql_text, t_pagination, t_cache_used, t_valkey_down, t_postgres_down] +OUTAGE = {t_valkey_down, t_postgres_down, t_expiry} + + +def main(): + if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') + p = argparse.ArgumentParser(description='Собственные проверки lab2 (уровень 4)') + p.add_argument('--base-url', default='http://127.0.0.1:8080') + p.add_argument('--short-ttl-url', default='http://127.0.0.1:8081') + p.add_argument('--skip-outages', action='store_true') + p.add_argument('--json', help='записать сводку в файл') + a = p.parse_args() + c = Client(a.base_url, timeout=15) + wait_ready(c) + failed = 0 + for t in TESTS: + if a.skip_outages and t in OUTAGE: + continue + start = time.monotonic() + try: + note = t(c, a) or '' + status = 'PASS' + except Failure as e: + note, status = str(e), 'FAIL' + failed += 1 + dur = time.monotonic() - start + RESULTS.append({'test': t.test_name, 'status': status, 'seconds': round(dur, 2), 'note': note}) + print('%-4s %6.2fs %s%s' % (status, dur, t.test_name, (' [' + note + ']') if note else '')) + if a.json: + Path(a.json).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print('%d/%d passed' % (len(RESULTS) - failed, len(RESULTS))) + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main() -- 2.54.0