From cbb92b7aefe6016d2b09cb2f22a9ce524a435746 Mon Sep 17 00:00:00 2001 From: Quzzqq Date: Fri, 23 Oct 2026 16:31:42 +0300 Subject: [PATCH] 2 lab --- .dockerignore | 1 + .env.example | 18 +- .github/workflows/app.yml | 37 +++ .gitignore | 1 + Dockerfile | 23 ++ Makefile | 5 +- REPORT.md | 111 ++++++- cmd/messenger/main.go | 233 +++++++++++++ compose.yaml | 95 ++++++ evidence/deps.txt | 6 + evidence/expiry.txt | 4 + evidence/persistence.txt | 5 + evidence/smoke.txt | Bin 0 -> 498 bytes evidence/student-tests.txt | Bin 0 -> 648 bytes evidence/track2-explain.txt | 77 +++++ evidence/track2.json | 99 ++++++ go.mod | 20 ++ go.sum | 40 +++ internal/authpw/password.go | 95 ++++++ internal/authpw/password_test.go | 19 ++ internal/httpapi/handlers.go | 253 ++++++++++++++ internal/httpapi/json.go | 186 +++++++++++ internal/httpapi/server.go | 224 +++++++++++++ internal/jsonstrict/strict.go | 108 ++++++ internal/migrate/migrate.go | 81 +++++ internal/model/model.go | 167 ++++++++++ internal/pgstore/store.go | 357 ++++++++++++++++++++ internal/sessions/redis.go | 85 +++++ migrations/001_tables.sql | 38 +++ migrations/002_message_history_index.sql | 1 + postman/lab2.postman_collection.json | 211 ++++++++++++ scripts/restart.sh | 4 + student-tests/deps.py | 73 ++++ student-tests/expiry.py | 28 ++ student-tests/run.py | 181 ++++++++++ student-tests/track2.py | 405 +++++++++++++++++++++++ 36 files changed, 3268 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/app.yml create mode 100644 Dockerfile create mode 100644 cmd/messenger/main.go create mode 100644 compose.yaml create mode 100644 evidence/deps.txt create mode 100644 evidence/expiry.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/track2-explain.txt create mode 100644 evidence/track2.json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/authpw/password.go create mode 100644 internal/authpw/password_test.go create mode 100644 internal/httpapi/handlers.go create mode 100644 internal/httpapi/json.go create mode 100644 internal/httpapi/server.go create mode 100644 internal/jsonstrict/strict.go create mode 100644 internal/migrate/migrate.go create mode 100644 internal/model/model.go create mode 100644 internal/pgstore/store.go create mode 100644 internal/sessions/redis.go create mode 100644 migrations/001_tables.sql create mode 100644 migrations/002_message_history_index.sql create mode 100644 postman/lab2.postman_collection.json create mode 100644 scripts/restart.sh create mode 100644 student-tests/deps.py create mode 100644 student-tests/expiry.py create mode 100644 student-tests/run.py create mode 100644 student-tests/track2.py diff --git a/.dockerignore b/.dockerignore index bc16e55..b9645df 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,5 @@ backups __pycache__ node_modules evidence +kak-sdano.md .DS_Store diff --git a/.env.example b/.env.example index 714c807..0bc6033 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,15 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Copy to .env for the local synthetic stand. Do not commit .env. +# Passwords below are local lab placeholders, not production secrets. +COMPOSE_PROJECT_NAME=mtusilab2 APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +SESSION_STORE=redis +POSTGRES_USER=messenger +POSTGRES_PASSWORD=labpgpass +POSTGRES_DB=messenger +REDIS_PASSWORD=labredispass +DATABASE_URL=postgres://messenger:labpgpass@postgres:5432/messenger?sslmode=disable +CACHE_URL=redis://:labredispass@redis:6379/0 +MIGRATIONS_DIR=/migrations diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 0000000..855d9c9 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,37 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Local env from the published example + run: cp .env.example .env + - name: Start the stand + run: docker compose --env-file .env up -d --build + - name: Wait until ready + run: | + python3 - <<'PY' + import time + import urllib.request + for _ in range(60): + try: + with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) as response: + if response.status == 200: + raise SystemExit(0) + except SystemExit: + raise + except Exception: + time.sleep(1) + raise SystemExit("server did not become ready") + PY + - name: Public smoke and student tests + run: make test student-test PYTHON=python3 + - name: Stop + if: always() + run: docker compose --env-file .env down diff --git a/.gitignore b/.gitignore index b5a7189..63ffc92 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +kak-sdano.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..23aa560 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.26.5-alpine@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd ./cmd +COPY internal ./internal +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/messenger + +FROM alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b + +RUN addgroup -g 10001 -S app && adduser -S -D -H -u 10001 -G app app +COPY --from=build /out/messenger /messenger +COPY migrations /migrations +USER 10001:10001 +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + MIGRATIONS_DIR=/migrations \ + SESSION_TTL_SECONDS=3600 +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=10 CMD ["/messenger", "healthcheck"] +ENTRYPOINT ["/messenger"] diff --git a/Makefile b/Makefile index 9ecb223..7f02bdc 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence student-test help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,3 +24,6 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +student-test: + $(PYTHON) student-tests/run.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index e42f2b0..6309797 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,118 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Черненко Сергей БПИ-2301 +- Целевая оценка; выбранный трек 5: 5, трек 2 (план запроса и восстановление) +- Commit/PR: заполнить после публикации +- Использованные источники, библиотеки и помощь ИИ: Go 1.26.5, pgx v5.7.6, go-redis v9.14.1, golang.org/x/crypto (Argon2id), PostgreSQL 16.10, Redis 7.4.6. Контракт и публичные тесты — из skeleton. Часть кода, тестов и этого текста написана с помощью ИИ в Cursor; команды и цифры ниже перепроверены прогоном 23 октября 2026. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker и Python 3.9+ (здесь Python 3.12.10). Go на хосте нужен только для `go test`. + +```powershell +Copy-Item .env.example .env +docker compose --env-file .env up -d --build +``` + +Подождать, пока `http://127.0.0.1:8080/health/ready` отвечает 200. Пароли в `.env.example` учебные, файл `.env` не коммитится. + +```powershell +python tests/smoke.py --base-url http://127.0.0.1:8080 +python student-tests/run.py --base-url http://127.0.0.1:8080 +python student-tests/deps.py --base-url http://127.0.0.1:8080 +python student-tests/track2.py +``` + +Проверка истечения сессии: в `.env` поставить `SESSION_TTL_SECONDS=2`, выполнить `docker compose --env-file .env up -d --no-deps --force-recreate api`, затем `python student-tests/expiry.py --base-url http://127.0.0.1:8080 --wait 3`. После этого вернуть `3600` и пересоздать `api`. Обычный smoke требует TTL не меньше 300. + +Сохранность на Linux: + +```sh +chmod +x scripts/restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +На Windows тот же `restart.sh` из Python не запускается. Прогон 23 октября выполнен в контейнере Alpine, подключённом к сети стенда. Команда записана в `evidence/persistence.txt`. + +Остановка без удаления данных: `docker compose --env-file .env down`. Удаление учебных volumes: `docker compose --env-file .env down -v`. + +Образы: `golang:1.26.5-alpine` и `alpine:3.24.1` закреплены digest в Dockerfile. PostgreSQL `postgres:16.10-alpine` (`sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297`). Redis `redis:7.4.6-alpine` (`sha256:3b73847e72874be07e6657b129a94761662b79bc0f679273757d4218573b2a98`). Собранный API: `sha256:149cd36e705b568cfbf00b8b889cba3772e1f0910d0bba514b811f0411b99685`. Процесс слушает `0.0.0.0:8080` от пользователя `10001:10001`. Лимит контейнера API: 2 CPU и 768 МиБ. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент ходит только в API. PostgreSQL — источник истины для пользователей, чатов, состава и сообщений. Redis хранит сессии и больше ничего. Пароль в Redis не попадает. В ключе лежит SHA-256 токена, не сам токен. TTL выставляется один раз при входе и чтением не продлевается. + +Пароль хешируется Argon2id (`m=19456` КиБ, `t=1`, `p=1`). Это password KDF, не checksum: проверка намеренно дороже, чтобы перебор пароля был медленным. SHA-256 для этого не годится. + +Создание чата — одна транзакция: строка чата и строки участников. Если участника нет, транзакция откатывается, пустой чат не остаётся. Отправка сообщения вставляет строку только если отправитель есть в `chat_members`. + +История читается keyset-запросом `(created_at, id) > cursor ORDER BY created_at, id LIMIT n+1`. Индекс `messages_history_idx` по `(chat_id, created_at, id)` создаётся миграцией `002`. Пул PostgreSQL — не больше 8 соединений, `statement_timeout` 8 с, таймаут запроса 4 с. + +`/health/live` проверяет только процесс. `/health/ready` — 200, если отвечают и PostgreSQL, и Redis. Иначе `503` без текста SQL и адресов. Запрос без Bearer получает `401`, даже если Redis выключен: отсутствие сессии не превращается в доверие. + +Из лабораторной 1 перенесены контракт сообщений, строгий JSON и курсор. `X-User-Id` больше не выбирает пользователя. Регистрация требует `password`. Вход — `Basic` только на `POST /api/v1/auth/sessions`. + +| Переменная | Куда попадает | +| --- | --- | +| `APP_HOST`, `APP_PORT`, `APP_ENV` | процесс API | +| `DATABASE_URL` | пул PostgreSQL; в Compose собирается из `POSTGRES_*` | +| `CACHE_URL` | клиент Redis; в Compose собирается из `REDIS_PASSWORD` | +| `SESSION_TTL_SECONDS` | срок новой сессии, по умолчанию 3600 | +| `SESSION_STORE=redis` | сессии только в Redis; пользователи и сообщения остаются в PostgreSQL | +| `MIGRATIONS_DIR` | каталог `001_tables.sql` и `002_message_history_index.sql` | + +Повторный старт читает `schema_migrations` и не применяет уже записанные файлы, поэтому данные не дублируются. + +PostgreSQL запущен с `synchronous_commit=on` и `fsync=on`. Успешный ответ клиенту означает, что COMMIT этой транзакции уже подтверждён сервером. Redis: `appendonly yes`, `appendfsync everysec`, volume `redis-data`. Одного TTL без AOF было бы мало: рестарт контейнера стёр бы сессии. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS | `evidence/smoke.txt` | +| Пользователи, чаты, участники, сообщения в PostgreSQL; уникальный username; миграции | схема `migrations/`, повторный старт API не плодит строки | smoke создаёт данные после миграции с пустого volume | `migrations/` | +| Атомарное создание чата | неизвестный участник, затем пустой список чатов | `400`, чат не виден | `evidence/student-tests.txt` | +| Basic только на вход, Bearer дальше, `X-User-Id` не даёт доступ, logout | smoke и `student-tests/run.py` | `401` / `204` / повторный `401` | `evidence/smoke.txt`, `evidence/student-tests.txt` | +| Argon2id | регистрация и проверка пароля | неверный и неизвестный пользователь дают один и тот же текст `401` | `evidence/student-tests.txt` | +| Рестарт API, PostgreSQL и Redis с прежним токеном | `scripts/restart.sh` через `tests/resilience.py` | PASS, сообщение и сессия на месте | `evidence/persistence.txt` | +| Redis — хранилище сессий с TTL и AOF | `CONFIG GET appendonly` внутри track2 | `yes`, `appendfsync everysec` | `evidence/track2.json` | +| Гонка username | 8 параллельных регистраций | ровно один `201`, остальные `409` | `evidence/student-tests.txt` | +| Пул, таймауты, параметризованные запросы | код `cmd/messenger/main.go`, запросы с `$1` | недоступная БД отвечает `503`, не зависает дольше таймаута клиента | `evidence/deps.txt` | +| EXPLAIN на чате ≥ 10 000 | keyset первой страницы чата из 20 000 | Index Scan по `messages_history_idx`, 0.191 мс, shared hit=14 | `evidence/track2-explain.txt` | +| Пагинация без полного прохода | глубокая страница чата из 70 000 | index scan читает 51 строку, не весь чат | `evidence/track2-explain.txt` | +| Короткий TTL, logout, неверный пароль | `expiry.py`, `run.py` | через 3 с после TTL=2 ответ `401`; logout `204` | `evidence/expiry.txt`, `evidence/student-tests.txt` | +| Падение Redis и PostgreSQL | `student-tests/deps.py` | ready `503`; без токена `401`; с токеном при мёртвом Redis `503` | `evidence/deps.txt` | +| Трек 2 | `python student-tests/track2.py` | 100040 сообщений, два плана, цена записи, restore | `evidence/track2.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза: страница истории через `(created_at, id) > cursor` на индексе `(chat_id, created_at, id)` читает около одной страницы даже глубоко в большом чате. `OFFSET` на том же индексе читает весь пропущенный префикс. Индекс только по `created_at` на отметке, до которой дошёл лишь самый большой чат, тоже почти точечный, поэтому он не доказывает отказ от фильтра по чату. Дополнительный индекс замедляет вставку. `pg_dump` во второй volume повторяет снимок и не содержит строку, закоммиченную после старта дампа. + +Стенд 23 октября 2026. Windows 11 (10.0.22621), 16 логических CPU, 15.3 ГиБ RAM, Docker 29.3.1. API ограничен 2 CPU и 768 МиБ. PostgreSQL и Redis в этом прогоне без отдельного CPU-лимита. Набор: 100040 сообщений, 73 чата, размеры 70000, 20000, 8000, 1500, 400, 80, 15, 5, сорок чатов по одному сообщению и 25 пустых. + +| Запрос | Что сделал планировщик | Время | Буферы | +| --- | --- | --- | --- | +| Keyset, глубина 60000, индекс истории | Index Scan, фактически 51 строка | 0.262 мс | shared hit=14 | +| OFFSET 60000 того же чата | тот же индекс, фактически 60051 строка | 30.490 мс | shared hit=1124 | +| Первая страница чата из 20000 | Index Scan по индексу истории | 0.191 мс | shared hit=14 | +| Тот же глубокий keyset, индекс только по `created_at` | Index Scan + фильтр, отброшена 1 строка | 0.391 мс | shared hit=11 read=3 | + +Вставка одних и тех же 50000 строк: только первичный ключ 118 мс, первичный ключ плюс индекс истории 153 мс. Это один прогон, не серия повторов. + +`pg_dump -Fc` занял 0.466 с и 2.98 МиБ. `pg_restore` в отдельный контейнер `postgres-restore` со своим volume — 0.667 с. Счётчики после восстановления совпали со снимком: 2 пользователя, 73 чата, 146 участий, 100040 сообщений. Три выбранные строки большого чата совпали. Строка, вставленная в источник после дампа, в восстановленной базе отсутствует (`extra_row_in_restore = 0`), а в источнике сообщений стало 100041. + +PostgreSQL в этом прогоне: `server_version=16.10`, `synchronous_commit=on`, `fsync=on`. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `restart` API, PostgreSQL и Redis | тот же токен читает то же сообщение | PASS публичного persistence-теста | контейнеры поднялись в пределах ожидания теста; volume не удалялся | +| Останов Redis | нет доступа «наугад» | без токена `401`, с токеном `503`, ready `503` | после `start` ready снова 200, старая сессия жива | +| Останов PostgreSQL | регистрация не проходит | `503`, аноним остаётся `401` | после `start` ready 200 | +| COMMIT после начала `pg_dump` | этой строки нет в новом хранилище | в источнике 100041, в копии 100040 | дамп 0.466 с, restore 0.667 с; строка после дампа потеряна для этой копии | +| Жёсткое выключение питания | не проверялось | нет измерения | при `appendfsync everysec` Redis может потерять около секунды новых сессий; выключение хоста до fsync PostgreSQL этим стендом не воспроизводилось | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Рестарт контейнеров на одном Docker-host не равен потере машины. `down -v` уничтожает и сообщения, и сессии. Снимок `pg_dump` не содержит транзакции, закоммиченные после его старта. Два сообщения в одну микросекунду упорядочиваются по `id`, а не по порядку COMMIT. Индекс только по времени на глубокой странице самого большого чата оказался почти таким же дешёвым: соседние чаты короче и до этой секунды не доходят. Вывод про OFFSET от этого не зависит: он прочитал 60051 строку против 51 у keyset. + +Пароль Argon2id с `t=1` — учебный параметр, не настройка lab7. TLS нет: стенд слушает только `127.0.0.1`. Redis без пароля с хоста не опубликован, пароль есть на внутренней сети. diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go new file mode 100644 index 0000000..b2c2cd8 --- /dev/null +++ b/cmd/messenger/main.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "mtusi.local/messenger/internal/authpw" + "mtusi.local/messenger/internal/httpapi" + "mtusi.local/messenger/internal/migrate" + "mtusi.local/messenger/internal/pgstore" + "mtusi.local/messenger/internal/sessions" +) + +type config struct { + host string + port string + env string + database string + cache string + migrations string + ttl time.Duration +} + +func main() { + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + if len(os.Args) > 1 && os.Args[1] == "healthcheck" { + os.Exit(healthcheck()) + } + cfg, err := loadConfig() + if err != nil { + logger.Error("config", "err", err.Error()) + os.Exit(1) + } + ctx := context.Background() + pool, err := waitPostgres(ctx, cfg.database, logger) + if err != nil { + logger.Error("postgres", "err", "not ready") + os.Exit(1) + } + defer pool.Close() + if err := migrate.Apply(ctx, pool, cfg.migrations); err != nil { + logger.Error("migrate", "err", "failed") + os.Exit(1) + } + cache, err := waitRedis(ctx, cfg.cache, logger) + if err != nil { + logger.Error("redis", "err", "not ready") + os.Exit(1) + } + defer cache.Close() + dummy, err := authpw.Hash("not-a-user-password") + if err != nil { + logger.Error("password", "err", "kdf init failed") + os.Exit(1) + } + api := httpapi.New(pgstore.New(pool), sessions.New(cache, cfg.ttl), logger, dummy) + server := &http.Server{ + Addr: net.JoinHostPort(cfg.host, cfg.port), + Handler: api.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 20 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, + } + logger.Info("start", "addr", server.Addr, "app_env", cfg.env, "session_ttl_seconds", int(cfg.ttl.Seconds())) + errCh := make(chan error, 1) + go func() { + errCh <- server.ListenAndServe() + }() + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGTERM, os.Interrupt) + select { + case err := <-errCh: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("listen", "err", err.Error()) + os.Exit(1) + } + case sig := <-signals: + logger.Info("shutdown", "signal", sig.String()) + api.StartDraining() + shut, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + if err := server.Shutdown(shut); err != nil { + logger.Error("shutdown", "err", err.Error()) + os.Exit(1) + } + logger.Info("shutdown_complete") + } +} + +func loadConfig() (config, error) { + port := getenv("APP_PORT", "8080") + if port == "" { + return config{}, fmt.Errorf("APP_PORT is empty") + } + database, err := secret("DATABASE_URL", "DATABASE_URL_FILE") + if err != nil { + return config{}, fmt.Errorf("database url is missing") + } + cache, err := secret("CACHE_URL", "CACHE_URL_FILE") + if err != nil { + return config{}, fmt.Errorf("cache url is missing") + } + ttlSeconds, err := strconv.Atoi(getenv("SESSION_TTL_SECONDS", "3600")) + if err != nil || ttlSeconds < 1 || ttlSeconds > 86400*30 { + return config{}, fmt.Errorf("SESSION_TTL_SECONDS is invalid") + } + return config{ + host: getenv("APP_HOST", "0.0.0.0"), + port: port, + env: getenv("APP_ENV", "lab"), + database: database, + cache: cache, + migrations: getenv("MIGRATIONS_DIR", "migrations"), + ttl: time.Duration(ttlSeconds) * time.Second, + }, nil +} + +func secret(directKey, fileKey string) (string, error) { + if value := strings.TrimSpace(os.Getenv(directKey)); value != "" { + return value, nil + } + path := strings.TrimSpace(os.Getenv(fileKey)) + if path == "" { + return "", fmt.Errorf("missing") + } + body, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(body)) + if value == "" { + return "", fmt.Errorf("empty") + } + return value, nil +} + +func getenv(key, fallback string) string { + value, ok := os.LookupEnv(key) + if !ok || value == "" { + return fallback + } + return value +} + +func waitPostgres(ctx context.Context, url string, logger *slog.Logger) (*pgxpool.Pool, error) { + var last error + for attempt := 1; attempt <= 30; attempt++ { + pool, err := openPool(url) + if err == nil { + ping, cancel := context.WithTimeout(ctx, 2*time.Second) + err = pool.Ping(ping) + cancel() + if err == nil { + return pool, nil + } + pool.Close() + } + last = err + logger.Info("postgres_wait", "attempt", attempt) + time.Sleep(time.Second) + } + return nil, last +} + +func openPool(url string) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(url) + if err != nil { + return nil, err + } + cfg.MaxConns = 8 + cfg.MinConns = 0 + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = time.Minute + cfg.HealthCheckPeriod = 5 * time.Second + cfg.ConnConfig.ConnectTimeout = time.Second + cfg.ConnConfig.RuntimeParams["statement_timeout"] = "8000" + cfg.ConnConfig.RuntimeParams["lock_timeout"] = "2000" + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = "5000" + return pgxpool.NewWithConfig(context.Background(), cfg) +} + +func waitRedis(ctx context.Context, url string, logger *slog.Logger) (*redis.Client, error) { + opt, err := redis.ParseURL(url) + if err != nil { + return nil, err + } + opt.DialTimeout = 2 * time.Second + opt.ReadTimeout = 2 * time.Second + opt.WriteTimeout = 2 * time.Second + opt.PoolSize = 8 + client := redis.NewClient(opt) + var last error + for attempt := 1; attempt <= 30; attempt++ { + ping, cancel := context.WithTimeout(ctx, 2*time.Second) + last = client.Ping(ping).Err() + cancel() + if last == nil { + return client, nil + } + logger.Info("redis_wait", "attempt", attempt) + time.Sleep(time.Second) + } + _ = client.Close() + return nil, last +} + +func healthcheck() int { + port := getenv("APP_PORT", "8080") + client := http.Client{Timeout: 2 * time.Second} + response, err := client.Get("http://127.0.0.1:" + port + "/health/ready") + if err != nil { + return 1 + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return 1 + } + return 0 +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..1de91ca --- /dev/null +++ b/compose.yaml @@ -0,0 +1,95 @@ +name: mtusilab2 + +services: + postgres: + image: postgres:16.10-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + command: ["postgres", "-c", "synchronous_commit=on", "-c", "fsync=on", "-c", "max_connections=50"] + volumes: + - pg-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + networks: [lab] + restart: unless-stopped + + redis: + image: redis:7.4.6-alpine + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD} + command: + [ + "redis-server", + "--appendonly", + "yes", + "--appendfsync", + "everysec", + "--requirepass", + "${REDIS_PASSWORD}", + ] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_PASSWORD\" ping | grep -q PONG"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 3s + networks: [lab] + restart: unless-stopped + + api: + build: + context: . + image: mtusi-messenger:lab2 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable + CACHE_URL: redis://:${REDIS_PASSWORD}@redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + MIGRATIONS_DIR: /migrations + ports: + - "127.0.0.1:8080:8080" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + mem_limit: 768m + cpus: 2 + networks: [lab] + restart: unless-stopped + + postgres-restore: + profiles: ["restore"] + image: postgres:16.10-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + command: ["postgres", "-c", "synchronous_commit=on", "-c", "fsync=on"] + volumes: + - pg-restore-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + networks: [lab] + +volumes: + pg-data: + redis-data: + pg-restore-data: + +networks: + lab: diff --git a/evidence/deps.txt b/evidence/deps.txt new file mode 100644 index 0000000..48c1551 --- /dev/null +++ b/evidence/deps.txt @@ -0,0 +1,6 @@ +PASS: dependency failures stay untrusted +Command: python student-tests/deps.py --base-url http://127.0.0.1:8080 +Redis stopped: /health/ready 503, request without a token 401, request with a token 503. +Postgres stopped: /health/ready 503, anonymous 401, registration 503. +Both services were started again and /health/ready returned 200. +Date: 2026-10-23 diff --git a/evidence/expiry.txt b/evidence/expiry.txt new file mode 100644 index 0000000..3f8d02f --- /dev/null +++ b/evidence/expiry.txt @@ -0,0 +1,4 @@ +PASS: session expired after 3.0 s +API was recreated with SESSION_TTL_SECONDS=2, then restored to 3600. +Command: python student-tests/expiry.py --base-url http://127.0.0.1:8080 --wait 3 +Date: 2026-10-23 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..d48aa6c --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,5 @@ +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +Command, from a Linux container joined to the compose network, because Windows Python cannot execute scripts/restart.sh (WinError 193): +docker run --rm --network mtusilab2_lab -v /var/run/docker.sock:/var/run/docker.sock -v "$PWD:/work" -w /work alpine:3.24.1 sh -c "apk add --no-cache python3 docker-cli docker-cli-compose && chmod +x scripts/restart.sh && python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://api:8080" +The hook restarts api, postgres and redis and does not remove volumes. +Date: 2026-10-23 diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000000000000000000000000000000000000..a066c04d9d0e3b45f05a6ea80fc4264bada48506 GIT binary patch literal 498 zcmXw#OKyWe5JYS3Q*Z%dgN^xQWhI;JEae0c28;rZWpJe2KCe4Oqw)0Tb#3jR6l*vs7NuV zVb+vm_cv0#z%S@j%xsnL-*a~D)|*f-$n0+Px_YtADV0=5loFztcn|Uaod$Q=v&q<* zEj2_2U%?+#@{L+1ju1KX8P2R*Q16I4YCGP-Z{fUo-$Ump1yqbFeS7Py^)pwuxnv~{ M9yO4q>u0$91^kdq`v3p{ literal 0 HcmV?d00001 diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 0000000000000000000000000000000000000000..d4b243af2185e7a24c9afffc4e48e803471d74a2 GIT binary patch literal 648 zcmZva%}&EG5QOL4r|<(1O2S{&8-fcQ00|*+4Q*(vhEO>zRo|X(wgnC#%g%a#W@pEF z)RmrCVJ1(lyMbNO9Wy=X zUPst{aLk}5@ZGo9HnOJ?pTc1UhXu3N!8_qGyA10(FpV=h)MtDxs|ntj?~Gcj=g1n{ zikR6J+X|)dG<+-gm(g|cqx8P7H8_iCI?Q4g9qM;fZ#jQiCHnXk+#v_{4Ew*?UQqb? zFiWA^s9C_*m}b6Lpyf7WZ(!?5{qmk91-fVTd-+drw9+}KR@zF|RC=-{=Po*Z>ui6| X`wesxXifYzR-n=a2qjj5eM-g`F$``) literal 0 HcmV?d00001 diff --git a/evidence/track2-explain.txt b/evidence/track2-explain.txt new file mode 100644 index 0000000..61985d1 --- /dev/null +++ b/evidence/track2-explain.txt @@ -0,0 +1,77 @@ +===== keyset_deep_history_index ===== +QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=1.16..39.32 rows=50 width=110) (actual time=0.151..0.163 rows=50 loops=1) + Buffers: shared hit=14 + -> Incremental Sort (cost=1.16..5285.82 rows=6926 width=110) (actual time=0.150..0.159 rows=50 loops=1) + Sort Key: created_at, ((id)::text) + Presorted Key: created_at + Full-sort Groups: 2 Sort Method: quicksort Average Memory: 33kB Peak Memory: 33kB + Buffers: shared hit=14 + -> Index Scan using messages_history_idx on messages (cost=0.42..4980.14 rows=6926 width=110) (actual time=0.046..0.062 rows=51 loops=1) + Index Cond: ((chat_id = 'a65fabe7-45b1-48f8-9e69-1ce146591e08'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:01:00.001+00'::timestamp with time zone, '4040726d-3600-430a-adda-fe4b752b1044'::uuid))) + Buffers: shared hit=5 + Planning: + Buffers: shared hit=127 + Planning Time: 0.918 ms + Execution Time: 0.262 ms +(14 rows) + +===== offset_deep_same_chat ===== +QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=9985.13..9993.45 rows=50 width=110) (actual time=30.434..30.447 rows=50 loops=1) + Buffers: shared hit=1124 + -> Incremental Sort (cost=0.61..11646.06 rows=69981 width=110) (actual time=0.143..29.165 rows=60050 loops=1) + Sort Key: created_at, ((id)::text) + Presorted Key: created_at + Full-sort Groups: 1877 Sort Method: quicksort Average Memory: 33kB Peak Memory: 33kB + Buffers: shared hit=1124 + -> Index Scan using messages_history_idx on messages (cost=0.42..9159.61 rows=69981 width=110) (actual time=0.045..20.330 rows=60051 loops=1) + Index Cond: (chat_id = 'a65fabe7-45b1-48f8-9e69-1ce146591e08'::uuid) + Buffers: shared hit=1115 + Planning: + Buffers: shared hit=128 + Planning Time: 0.912 ms + Execution Time: 30.490 ms +(14 rows) + +===== keyset_first_page_at_least_10000 ===== +QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.74..17.47 rows=50 width=110) (actual time=0.139..0.153 rows=50 loops=1) + Buffers: shared hit=14 + -> Incremental Sort (cost=0.74..6679.62 rows=19958 width=110) (actual time=0.139..0.150 rows=50 loops=1) + Sort Key: created_at, ((id)::text) + Presorted Key: created_at + Full-sort Groups: 2 Sort Method: quicksort Average Memory: 33kB Peak Memory: 33kB + Buffers: shared hit=14 + -> Index Scan using messages_history_idx on messages (cost=0.42..5831.91 rows=19958 width=110) (actual time=0.043..0.059 rows=51 loops=1) + Index Cond: (chat_id = '09ba6f62-1a1f-449c-89ae-9ca655000b36'::uuid) + Buffers: shared hit=5 + Planning: + Buffers: shared hit=128 + Planning Time: 0.918 ms + Execution Time: 0.191 ms +(14 rows) + +===== keyset_deep_created_at_index_only ===== +QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (cost=1.04..39.19 rows=50 width=110) (actual time=0.212..0.257 rows=50 loops=1) + Buffers: shared hit=11 read=3 + -> Incremental Sort (cost=1.04..5352.98 rows=7015 width=110) (actual time=0.211..0.252 rows=50 loops=1) + Sort Key: created_at, ((id)::text) + Presorted Key: created_at + Full-sort Groups: 2 Sort Method: quicksort Average Memory: 33kB Peak Memory: 33kB + Buffers: shared hit=11 read=3 + -> Index Scan using messages_time_created on messages_time (cost=0.29..5043.54 rows=7015 width=110) (actual time=0.067..0.116 rows=51 loops=1) + Index Cond: (created_at >= '2026-01-01 00:01:00.001+00'::timestamp with time zone) + Filter: ((chat_id = 'a65fabe7-45b1-48f8-9e69-1ce146591e08'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:01:00.001+00'::timestamp with time zone, '4040726d-3600-430a-adda-fe4b752b1044'::uuid))) + Rows Removed by Filter: 1 + Buffers: shared hit=2 read=3 + Planning: + Buffers: shared hit=106 read=1 + Planning Time: 1.195 ms + Execution Time: 0.391 ms +(16 rows) diff --git a/evidence/track2.json b/evidence/track2.json new file mode 100644 index 0000000..b673126 --- /dev/null +++ b/evidence/track2.json @@ -0,0 +1,99 @@ +{ + "track": 2, + "hypothesis": "Keyset on (chat_id, created_at, id) reads about one page even deep in a large chat. OFFSET on the same index walks the skipped prefix. An index on created_at alone is also selective at a timestamp that only the largest chat has reached, so it is not a substitute for bounding the walk by chat_id. The extra history index increases insert time. A custom-format dump restored into a second Postgres volume matches the snapshot and omits a row committed after pg_dump started.", + "hardware": { + "platform": "Windows-11-10.0.22621-SP0", + "cpu_count": 16, + "ram_gib": 15.3, + "api_limit": "2 CPU, 768 MiB" + }, + "postgres": { + "image": "postgres:16.10-alpine", + "version": "16.10", + "synchronous_commit": "on", + "fsync": "on" + }, + "redis_persistence": "appendonly\nyes\nappendfsync\neverysec", + "distribution": { + "messages": 100040, + "chats": 73, + "largest_chat": 70000, + "empty_chats": 25, + "sizes": [ + 70000, + 20000, + 8000, + 1500, + 400, + 80, + 15, + 5, + 1, + 0 + ] + }, + "queries": { + "keyset_deep_history_index": { + "execution_ms": 0.262, + "buffers": "", + "shared_hit": 14, + "shared_read": 0, + "uses_history_index": true, + "node": "Index Scan" + }, + "offset_deep_same_chat": { + "execution_ms": 30.49, + "buffers": "", + "shared_hit": 1124, + "shared_read": 0, + "uses_history_index": true, + "node": "Index Scan" + }, + "keyset_first_page_at_least_10000": { + "execution_ms": 0.191, + "buffers": "", + "shared_hit": 14, + "shared_read": 0, + "uses_history_index": true, + "node": "Index Scan" + }, + "keyset_deep_created_at_index_only": { + "execution_ms": 0.391, + "buffers": "", + "shared_hit": 11, + "shared_read": 0, + "uses_history_index": false, + "node": "Index Scan" + } + }, + "write": { + "rows": 50000, + "primary_key_ms": 118.0, + "primary_key_and_history_index_ms": 153.0 + }, + "backup": { + "dump_seconds": 0.466, + "restore_seconds": 0.667, + "dump_bytes": 2980396, + "counts_before_dump": { + "users": 2, + "chats": 73, + "chat_members": 146, + "messages": 100040 + }, + "counts_source_after_extra_commit": { + "users": 2, + "chats": 73, + "chat_members": 146, + "messages": 100041 + }, + "counts_restored": { + "users": 2, + "chats": 73, + "chat_members": 146, + "messages": 100040 + }, + "sample_match": true, + "extra_row_in_restore": 0 + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e743e48 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module mtusi.local/messenger + +go 1.26 + +require ( + github.com/jackc/pgx/v5 v5.7.6 + github.com/redis/go-redis/v9 v9.14.1 + golang.org/x/crypto v0.42.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..684080e --- /dev/null +++ b/go.sum @@ -0,0 +1,40 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ= +github.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/authpw/password.go b/internal/authpw/password.go new file mode 100644 index 0000000..fb8bee0 --- /dev/null +++ b/internal/authpw/password.go @@ -0,0 +1,95 @@ +package authpw + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "fmt" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + memoryKiB = 19 * 1024 + timeCost = 1 + threads = 1 + keyLen = 32 + saltLen = 16 +) + +func Hash(password string) (string, error) { + salt := make([]byte, saltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + sum := argon2.IDKey([]byte(password), salt, timeCost, memoryKiB, threads, keyLen) + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, memoryKiB, timeCost, threads, + base64.RawStdEncoding.EncodeToString(salt), + base64.RawStdEncoding.EncodeToString(sum), + ), nil +} + +func Verify(password, encoded string) bool { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version { + return false + } + memory, time, parallelism, ok := parseParams(parts[3]) + if !ok { + return false + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil || len(want) == 0 { + return false + } + sum := argon2.IDKey([]byte(password), salt, time, memory, parallelism, uint32(len(want))) + return subtle.ConstantTimeCompare(sum, want) == 1 +} + +func parseParams(raw string) (uint32, uint32, uint8, bool) { + var memory, time uint32 + var parallelism uint8 + for _, item := range strings.Split(raw, ",") { + key, value, ok := strings.Cut(item, "=") + if !ok { + return 0, 0, 0, false + } + switch key { + case "m": + n, err := strconv.ParseUint(value, 10, 32) + if err != nil || n == 0 || n > 1024*1024 { + return 0, 0, 0, false + } + memory = uint32(n) + case "t": + n, err := strconv.ParseUint(value, 10, 32) + if err != nil || n == 0 || n > 10 { + return 0, 0, 0, false + } + time = uint32(n) + case "p": + n, err := strconv.ParseUint(value, 10, 8) + if err != nil || n == 0 || n > 4 { + return 0, 0, 0, false + } + parallelism = uint8(n) + default: + return 0, 0, 0, false + } + } + if memory == 0 || time == 0 || parallelism == 0 { + return 0, 0, 0, false + } + return memory, time, parallelism, true +} diff --git a/internal/authpw/password_test.go b/internal/authpw/password_test.go new file mode 100644 index 0000000..7ef9bc0 --- /dev/null +++ b/internal/authpw/password_test.go @@ -0,0 +1,19 @@ +package authpw + +import "testing" + +func TestHashVerifiesOnlyTheSamePassword(t *testing.T) { + encoded, err := Hash("correct-horse-battery") + if err != nil { + t.Fatal(err) + } + if !Verify("correct-horse-battery", encoded) { + t.Fatal("same password rejected") + } + if Verify("correct-horse-batteries", encoded) { + t.Fatal("other password accepted") + } + if Verify("correct-horse-battery", "sha256:nope") { + t.Fatal("foreign encoding accepted") + } +} diff --git a/internal/httpapi/handlers.go b/internal/httpapi/handlers.go new file mode 100644 index 0000000..a60e801 --- /dev/null +++ b/internal/httpapi/handlers.go @@ -0,0 +1,253 @@ +package httpapi + +import ( + "errors" + "net/http" + "strings" + + "mtusi.local/messenger/internal/authpw" + "mtusi.local/messenger/internal/model" +) + +func (a *API) createUser(w http.ResponseWriter, r *http.Request) { + var body struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` + } + if err := readJSON(w, r, &body); err != nil { + a.writeReadError(w, r, err) + return + } + if !model.ValidUsername(body.Username) || !model.ValidLabel(body.DisplayName, 100) || !model.ValidPassword(body.Password) { + writeError(w, http.StatusBadRequest, "invalid_request", "Недопустимое имя или отображаемое имя", requestID(r)) + return + } + hash, err := authpw.Hash(body.Password) + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Внутренняя ошибка", requestID(r)) + return + } + user, err := a.db.CreateUser(r.Context(), body.Username, body.DisplayName, hash) + if err != nil { + writeStoreError(w, r, err, "Недопустимое имя или отображаемое имя") + return + } + writeJSON(w, http.StatusCreated, userToJSON(user)) +} + +func (a *API) createSession(w http.ResponseWriter, r *http.Request) { + if !bodyEmpty(r) { + writeError(w, http.StatusBadRequest, "invalid_request", "Неверный JSON", requestID(r)) + return + } + username, password, ok := basicCredentials(r) + if !ok { + writeUnauthorized(w, r, true) + return + } + user, hash, err := a.db.UserByUsername(r.Context(), username) + if errors.Is(err, model.ErrUnavailable) { + writeStoreError(w, r, err, "") + return + } + if err != nil { + user = model.User{} + hash = "" + } + if !model.ValidPassword(password) || !a.checkPassword(password, hash) || user.ID == "" { + writeUnauthorized(w, r, true) + return + } + token, expires, err := a.session.Create(r.Context(), user.ID) + if err != nil { + writeStoreError(w, r, err, "") + return + } + writeJSON(w, http.StatusCreated, sessionJSON{ + Token: token, + TokenType: "Bearer", + ExpiresAt: apiTime(expires), + User: userToJSON(user), + }) +} + +func (a *API) revokeSession(w http.ResponseWriter, r *http.Request) { + token, ok := bearerToken(r) + if !ok { + writeUnauthorized(w, r, false) + return + } + if _, err := a.session.Lookup(r.Context(), token); err != nil { + if errors.Is(err, model.ErrUnavailable) { + writeStoreError(w, r, err, "") + return + } + writeUnauthorized(w, r, false) + return + } + if err := a.session.Revoke(r.Context(), token); err != nil { + writeStoreError(w, r, err, "") + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (a *API) currentUser(w http.ResponseWriter, r *http.Request) { + user, ok := a.requireUser(w, r) + if !ok { + return + } + writeJSON(w, http.StatusOK, userToJSON(user)) +} + +func (a *API) createChat(w http.ResponseWriter, r *http.Request) { + user, ok := a.requireUser(w, r) + if !ok { + return + } + var body struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + } + if err := readJSON(w, r, &body); err != nil { + a.writeReadError(w, r, err) + return + } + chat, err := a.db.CreateChat(r.Context(), user.ID, body.Title, body.MemberIDs) + if err != nil { + writeStoreError(w, r, err, "Неверный список участников или название чата") + return + } + writeJSON(w, http.StatusCreated, chatToJSON(chat)) +} + +func (a *API) listChats(w http.ResponseWriter, r *http.Request) { + user, ok := a.requireUser(w, r) + if !ok { + return + } + limit, cursor, ok := a.pageQuery(w, r) + if !ok { + return + } + if cursor != nil && cursor.Resource != model.ResourceChats { + writeError(w, http.StatusBadRequest, "invalid_request", "Неверный cursor или limit", requestID(r)) + return + } + items, next, err := a.db.ListChats(r.Context(), user.ID, limit, cursor) + if err != nil { + writeStoreError(w, r, err, "Неверный cursor или limit") + return + } + page := chatPageJSON{Items: make([]chatJSON, 0, len(items)), NextCursor: next} + for _, item := range items { + page.Items = append(page.Items, chatToJSON(item)) + } + writeJSON(w, http.StatusOK, page) +} + +func (a *API) getChat(w http.ResponseWriter, r *http.Request) { + user, ok := a.requireUser(w, r) + if !ok { + return + } + chatID, ok := pathChatID(w, r) + if !ok { + return + } + chat, err := a.db.Chat(r.Context(), chatID, user.ID) + if err != nil { + writeStoreError(w, r, err, "Неверный идентификатор чата") + return + } + writeJSON(w, http.StatusOK, chatToJSON(chat)) +} + +func (a *API) createMessage(w http.ResponseWriter, r *http.Request) { + user, ok := a.requireUser(w, r) + if !ok { + return + } + chatID, ok := pathChatID(w, r) + if !ok { + return + } + if _, err := a.db.Chat(r.Context(), chatID, user.ID); err != nil { + writeStoreError(w, r, err, "Неверный идентификатор чата") + return + } + var body struct { + Text string `json:"text"` + } + if err := readJSON(w, r, &body); err != nil { + a.writeReadError(w, r, err) + return + } + msg, err := a.db.AddMessage(r.Context(), chatID, user.ID, body.Text) + if err != nil { + writeStoreError(w, r, err, "Недопустимый текст сообщения") + return + } + writeJSON(w, http.StatusCreated, messageToJSON(msg)) +} + +func (a *API) listMessages(w http.ResponseWriter, r *http.Request) { + user, ok := a.requireUser(w, r) + if !ok { + return + } + chatID, ok := pathChatID(w, r) + if !ok { + return + } + limit, cursor, ok := a.pageQuery(w, r) + if !ok { + return + } + if cursor != nil && (cursor.Resource != model.ResourceMessages || cursor.ChatID != chatID) { + writeError(w, http.StatusBadRequest, "invalid_request", "Неверный cursor или limit", requestID(r)) + return + } + items, next, err := a.db.ListMessages(r.Context(), chatID, user.ID, limit, cursor) + if err != nil { + writeStoreError(w, r, err, "Неверный cursor или limit") + return + } + page := messagePageJSON{Items: make([]messageJSON, 0, len(items)), NextCursor: next} + for _, item := range items { + page.Items = append(page.Items, messageToJSON(item)) + } + writeJSON(w, http.StatusOK, page) +} + +func (a *API) pageQuery(w http.ResponseWriter, r *http.Request) (int, *model.Cursor, bool) { + limit, err := parseLimit(r) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "Неверный cursor или limit", requestID(r)) + return 0, nil, false + } + cursor, err := parseCursor(r) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "Неверный cursor или limit", requestID(r)) + return 0, nil, false + } + return limit, cursor, true +} + +func (a *API) writeReadError(w http.ResponseWriter, r *http.Request, err error) { + if errors.Is(err, errTooLarge) { + writeError(w, http.StatusRequestEntityTooLarge, "payload_too_large", "Слишком большой запрос", requestID(r)) + return + } + writeError(w, http.StatusBadRequest, "invalid_request", "Неверный JSON", requestID(r)) +} + +func pathChatID(w http.ResponseWriter, r *http.Request) (string, bool) { + chatID := strings.ToLower(r.PathValue("chat_id")) + if !model.ValidUUID(chatID) { + writeError(w, http.StatusBadRequest, "invalid_request", "Неверный идентификатор чата", requestID(r)) + return "", false + } + return chatID, true +} diff --git a/internal/httpapi/json.go b/internal/httpapi/json.go new file mode 100644 index 0000000..7a628b6 --- /dev/null +++ b/internal/httpapi/json.go @@ -0,0 +1,186 @@ +package httpapi + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "time" + + "mtusi.local/messenger/internal/jsonstrict" + "mtusi.local/messenger/internal/model" +) + +type statusWriter struct { + http.ResponseWriter + status int + wrote bool +} + +func (w *statusWriter) WriteHeader(status int) { + if w.wrote { + return + } + w.status = status + w.wrote = true + w.ResponseWriter.WriteHeader(status) +} + +func (w *statusWriter) Write(body []byte) (int, error) { + if !w.wrote { + w.status = http.StatusOK + w.wrote = true + } + return w.ResponseWriter.Write(body) +} + +type apiTime time.Time + +func (t apiTime) MarshalJSON() ([]byte, error) { + return []byte(`"` + model.FormatTime(time.Time(t)) + `"`), nil +} + +type userJSON struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt apiTime `json:"created_at"` +} + +type chatJSON struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt apiTime `json:"created_at"` +} + +type messageJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt apiTime `json:"created_at"` +} + +type chatPageJSON struct { + Items []chatJSON `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type messagePageJSON struct { + Items []messageJSON `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type sessionJSON struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt apiTime `json:"expires_at"` + User userJSON `json:"user"` +} + +func userToJSON(user model.User) userJSON { + return userJSON{ + ID: user.ID, + Username: user.Username, + DisplayName: user.DisplayName, + CreatedAt: apiTime(user.CreatedAt), + } +} + +func chatToJSON(chat model.Chat) chatJSON { + members := chat.MemberIDs + if members == nil { + members = []string{} + } + return chatJSON{ + ID: chat.ID, + Title: chat.Title, + MemberIDs: members, + CreatedAt: apiTime(chat.CreatedAt), + } +} + +func messageToJSON(msg model.Message) messageJSON { + return messageJSON{ + ID: msg.ID, + ChatID: msg.ChatID, + SenderID: msg.SenderID, + Text: msg.Text, + CreatedAt: apiTime(msg.CreatedAt), + } +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(value) +} + +func writeError(w http.ResponseWriter, status int, code, message, requestID string) { + writeJSON(w, status, map[string]any{ + "error": map[string]string{ + "code": code, + "message": message, + "request_id": requestID, + }, + }) +} + +func writeUnauthorized(w http.ResponseWriter, r *http.Request, basic bool) { + if basic { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } else { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + writeError(w, http.StatusUnauthorized, "unauthorized", "Нет действительной идентификации", requestID(r)) +} + +func readJSON(w http.ResponseWriter, r *http.Request, dst any) error { + media, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || media != "application/json" { + return errBadJSON + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64*1024)) + if err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + return errTooLarge + } + return errBadJSON + } + if err := jsonstrict.Decode(body, dst); err != nil { + return errBadJSON + } + return nil +} + +func bodyEmpty(r *http.Request) bool { + if r.Body == nil { + return true + } + buf := make([]byte, 1) + n, err := r.Body.Read(buf) + if n > 0 { + return false + } + return err == io.EOF || err == nil +} + +func newRequestID() string { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return "request" + } + return hex.EncodeToString(raw[:]) +} + +var ( + errBadJSON = errors.New("bad json") + errTooLarge = errors.New("too large") +) diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go new file mode 100644 index 0000000..66a3963 --- /dev/null +++ b/internal/httpapi/server.go @@ -0,0 +1,224 @@ +package httpapi + +import ( + "context" + "encoding/base64" + "errors" + "log/slog" + "net/http" + "runtime/debug" + "strconv" + "strings" + "sync/atomic" + "time" + + "mtusi.local/messenger/internal/authpw" + "mtusi.local/messenger/internal/model" + "mtusi.local/messenger/internal/pgstore" + "mtusi.local/messenger/internal/sessions" +) + +type API struct { + db *pgstore.Store + session *sessions.Store + log *slog.Logger + dummy string + draining atomic.Bool +} + +func New(db *pgstore.Store, session *sessions.Store, logger *slog.Logger, dummyHash string) *API { + if logger == nil { + logger = slog.Default() + } + return &API{db: db, session: session, log: logger, dummy: dummyHash} +} + +func (a *API) StartDraining() { + a.draining.Store(true) +} + +func (a *API) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", a.live) + mux.HandleFunc("GET /health/ready", a.ready) + mux.HandleFunc("POST /api/v1/users", a.createUser) + mux.HandleFunc("GET /api/v1/users/me", a.currentUser) + mux.HandleFunc("POST /api/v1/auth/sessions", a.createSession) + mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.revokeSession) + mux.HandleFunc("POST /api/v1/chats", a.createChat) + mux.HandleFunc("GET /api/v1/chats", a.listChats) + mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.getChat) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.createMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.listMessages) + mux.HandleFunc("/", a.notFound) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + id := newRequestID() + ctx, cancel := context.WithTimeout(r.Context(), 4*time.Second) + defer cancel() + ctx = context.WithValue(ctx, requestIDKey{}, id) + r = r.WithContext(ctx) + sw := &statusWriter{ResponseWriter: w, status: http.StatusOK} + sw.Header().Set("X-Request-Id", id) + defer func() { + if rec := recover(); rec != nil { + a.log.Error("panic", "request_id", id, "method", r.Method, "path", r.URL.Path) + a.log.Error("panic_stack", "request_id", id, "stack", string(debug.Stack())) + if !sw.wrote { + writeError(sw, http.StatusInternalServerError, "internal_error", "Внутренняя ошибка", id) + } + } + a.log.Info("request", + "request_id", id, + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "duration_ms", float64(time.Since(started).Microseconds())/1000, + ) + }() + if a.draining.Load() && r.URL.Path != "/health/live" { + writeError(sw, http.StatusServiceUnavailable, "unavailable", "Сервер завершает работу", id) + return + } + mux.ServeHTTP(sw, r) + }) +} + +func (a *API) live(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (a *API) ready(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if err := a.db.Ping(ctx); err != nil || a.session.Ping(ctx) != nil { + writeError(w, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", requestID(r)) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (a *API) notFound(w http.ResponseWriter, r *http.Request) { + writeError(w, http.StatusNotFound, "not_found", "Объект не найден", requestID(r)) +} + +func (a *API) requireUser(w http.ResponseWriter, r *http.Request) (model.User, bool) { + token, ok := bearerToken(r) + if !ok { + writeUnauthorized(w, r, false) + return model.User{}, false + } + userID, err := a.session.Lookup(r.Context(), token) + if errors.Is(err, model.ErrUnavailable) { + writeError(w, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", requestID(r)) + return model.User{}, false + } + if err != nil { + writeUnauthorized(w, r, false) + return model.User{}, false + } + user, err := a.db.UserByID(r.Context(), userID) + if errors.Is(err, model.ErrUnavailable) { + writeError(w, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", requestID(r)) + return model.User{}, false + } + if err != nil { + writeUnauthorized(w, r, false) + return model.User{}, false + } + return user, true +} + +func bearerToken(r *http.Request) (string, bool) { + scheme, param, ok := splitAuth(r.Header.Get("Authorization")) + if !ok || !strings.EqualFold(scheme, "Bearer") || len(param) < 32 || strings.ContainsAny(param, " \t") { + return "", false + } + return param, true +} + +func basicCredentials(r *http.Request) (string, string, bool) { + scheme, param, ok := splitAuth(r.Header.Get("Authorization")) + if !ok || !strings.EqualFold(scheme, "Basic") { + return "", "", false + } + raw, err := base64.StdEncoding.DecodeString(param) + if err != nil { + return "", "", false + } + username, password, ok := strings.Cut(string(raw), ":") + if !ok { + return "", "", false + } + return username, password, true +} + +func splitAuth(header string) (string, string, bool) { + scheme, param, ok := strings.Cut(strings.TrimSpace(header), " ") + if !ok || scheme == "" || strings.TrimSpace(param) == "" { + return "", "", false + } + return scheme, strings.TrimSpace(param), true +} + +func parseLimit(r *http.Request) (int, error) { + values, ok := r.URL.Query()["limit"] + if !ok { + return 50, nil + } + if len(values) != 1 { + return 0, errBadQuery + } + n, err := strconv.Atoi(values[0]) + if err != nil || n < 1 || n > 100 { + return 0, errBadQuery + } + return n, nil +} + +func parseCursor(r *http.Request) (*model.Cursor, error) { + values, ok := r.URL.Query()["cursor"] + if !ok { + return nil, nil + } + if len(values) != 1 || values[0] == "" { + return nil, errBadQuery + } + cursor, err := model.DecodeCursor(values[0]) + if err != nil { + return nil, errBadQuery + } + return &cursor, nil +} + +func writeStoreError(w http.ResponseWriter, r *http.Request, err error, invalidMessage string) { + switch { + case errors.Is(err, model.ErrConflict): + writeError(w, http.StatusConflict, "conflict", "Имя пользователя уже занято", requestID(r)) + case errors.Is(err, model.ErrNotFound): + writeError(w, http.StatusNotFound, "not_found", "Объект не найден", requestID(r)) + case errors.Is(err, model.ErrUnavailable): + writeError(w, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", requestID(r)) + default: + writeError(w, http.StatusBadRequest, "invalid_request", invalidMessage, requestID(r)) + } +} + +func (a *API) checkPassword(password, encoded string) bool { + if encoded == "" { + encoded = a.dummy + } + return authpw.Verify(password, encoded) +} + +type requestIDKey struct{} + +func requestID(r *http.Request) string { + id, _ := r.Context().Value(requestIDKey{}).(string) + if id == "" { + id = newRequestID() + } + return id +} + +var errBadQuery = errors.New("bad query") diff --git a/internal/jsonstrict/strict.go b/internal/jsonstrict/strict.go new file mode 100644 index 0000000..45c4f73 --- /dev/null +++ b/internal/jsonstrict/strict.go @@ -0,0 +1,108 @@ +package jsonstrict + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "reflect" + "strings" + "unicode/utf8" +) + +var ErrInvalid = errors.New("invalid json") + +func Decode(body []byte, dst any) error { + if !utf8.Valid(body) { + return ErrInvalid + } + keys, err := objectKeys(body) + if err != nil { + return ErrInvalid + } + allowed := tagNames(dst) + for _, key := range keys { + if _, ok := allowed[key]; !ok { + return ErrInvalid + } + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + return ErrInvalid + } + var trailer struct{} + if err := dec.Decode(&trailer); !errors.Is(err, io.EOF) { + return ErrInvalid + } + return nil +} + +func objectKeys(body []byte) ([]string, error) { + dec := json.NewDecoder(bytes.NewReader(body)) + token, err := dec.Token() + if err != nil { + return nil, err + } + delim, ok := token.(json.Delim) + if !ok || delim != '{' { + return nil, ErrInvalid + } + seen := map[string]struct{}{} + var keys []string + for dec.More() { + keyToken, err := dec.Token() + if err != nil { + return nil, err + } + key, ok := keyToken.(string) + if !ok { + return nil, ErrInvalid + } + if _, dup := seen[key]; dup { + return nil, ErrInvalid + } + seen[key] = struct{}{} + keys = append(keys, key) + var skip json.RawMessage + if err := dec.Decode(&skip); err != nil { + return nil, err + } + } + token, err = dec.Token() + if err != nil { + return nil, err + } + delim, ok = token.(json.Delim) + if !ok || delim != '}' { + return nil, ErrInvalid + } + var trailer struct{} + if err := dec.Decode(&trailer); !errors.Is(err, io.EOF) { + return nil, ErrInvalid + } + return keys, nil +} + +func tagNames(dst any) map[string]struct{} { + value := reflect.TypeOf(dst) + if value == nil { + return nil + } + if value.Kind() == reflect.Pointer { + value = value.Elem() + } + names := make(map[string]struct{}) + if value.Kind() != reflect.Struct { + return names + } + for i := 0; i < value.NumField(); i++ { + tag := value.Field(i).Tag.Get("json") + name := strings.Split(tag, ",")[0] + if name == "" || name == "-" { + continue + } + names[name] = struct{}{} + } + return names +} diff --git a/internal/migrate/migrate.go b/internal/migrate/migrate.go new file mode 100644 index 0000000..5ba9d67 --- /dev/null +++ b/internal/migrate/migrate.go @@ -0,0 +1,81 @@ +package migrate + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func Apply(ctx context.Context, pool *pgxpool.Pool, dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("read migrations: %w", err) + } + var names []string + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sql") { + continue + } + names = append(names, entry.Name()) + } + sort.Strings(names) + if len(names) == 0 { + return fmt.Errorf("no sql migrations in %s", dir) + } + _, err = pool.Exec(ctx, ` + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + )`) + if err != nil { + return err + } + for _, name := range names { + var exists bool + if err := pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = $1)`, name).Scan(&exists); err != nil { + return err + } + if exists { + continue + } + body, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return err + } + tx, err := pool.Begin(ctx) + if err != nil { + return err + } + for _, stmt := range statements(string(body)) { + if _, err := tx.Exec(ctx, stmt); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("%s: %w", name, err) + } + } + if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + } + return nil +} + +func statements(body string) []string { + parts := strings.Split(body, ";") + out := make([]string, 0, len(parts)) + for _, part := range parts { + stmt := strings.TrimSpace(part) + if stmt != "" { + out = append(out, stmt) + } + } + return out +} diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..28ea5b6 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,167 @@ +package model + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "regexp" + "strings" + "time" + "unicode/utf8" + + "mtusi.local/messenger/internal/jsonstrict" +) + +const ( + ResourceMessages = "messages" + ResourceChats = "chats" +) + +var ( + ErrInvalid = errors.New("invalid") + ErrConflict = errors.New("conflict") + ErrNotFound = errors.New("not found") + ErrUnavailable = errors.New("unavailable") + + usernameRE = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + uuidRE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) +) + +type User struct { + ID string + Username string + DisplayName string + CreatedAt time.Time +} + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} + +type Message struct { + ID string + ChatID string + SenderID string + Text string + CreatedAt time.Time +} + +type Cursor struct { + Resource string + ChatID string + Time time.Time + ID string +} + +type cursorWire struct { + V int `json:"v"` + Resource string `json:"r"` + ChatID string `json:"c,omitempty"` + Time string `json:"t"` + ID string `json:"id"` +} + +func ValidUUID(value string) bool { + return uuidRE.MatchString(strings.ToLower(value)) +} + +func NormalizeUUID(value string) (string, bool) { + id := strings.ToLower(strings.TrimSpace(value)) + if !uuidRE.MatchString(id) { + return "", false + } + return id, true +} + +func ValidUsername(value string) bool { + return usernameRE.MatchString(value) +} + +func ValidLabel(value string, max int) bool { + if strings.TrimSpace(value) == "" { + return false + } + n := utf8.RuneCountInString(value) + return n >= 1 && n <= max +} + +func ValidPassword(value string) bool { + if strings.TrimSpace(value) == "" { + return false + } + n := utf8.RuneCountInString(value) + return n >= 12 && n <= 128 +} + +func FormatTime(value time.Time) string { + return value.UTC().Format("2006-01-02T15:04:05.000000Z") +} + +func NewID() (string, error) { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + raw[6] = (raw[6] & 0x0f) | 0x40 + raw[8] = (raw[8] & 0x3f) | 0x80 + enc := hex.EncodeToString(raw[:]) + return enc[0:8] + "-" + enc[8:12] + "-" + enc[12:16] + "-" + enc[16:20] + "-" + enc[20:32], nil +} + +func EncodeCursor(cursor Cursor) (string, error) { + raw, err := json.Marshal(cursorWire{ + V: 1, + Resource: cursor.Resource, + ChatID: cursor.ChatID, + Time: FormatTime(cursor.Time), + ID: cursor.ID, + }) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(raw), nil +} + +func DecodeCursor(token string) (Cursor, error) { + if token == "" { + return Cursor{}, ErrInvalid + } + raw, err := base64.RawURLEncoding.DecodeString(token) + if err != nil { + return Cursor{}, ErrInvalid + } + var payload cursorWire + if err := jsonstrict.Decode(raw, &payload); err != nil { + return Cursor{}, ErrInvalid + } + if payload.V != 1 || !ValidUUID(payload.ID) { + return Cursor{}, ErrInvalid + } + switch payload.Resource { + case ResourceMessages: + if !ValidUUID(payload.ChatID) { + return Cursor{}, ErrInvalid + } + case ResourceChats: + if payload.ChatID != "" { + return Cursor{}, ErrInvalid + } + default: + return Cursor{}, ErrInvalid + } + ts, err := time.Parse("2006-01-02T15:04:05.000000Z", payload.Time) + if err != nil { + return Cursor{}, ErrInvalid + } + return Cursor{ + Resource: payload.Resource, + ChatID: strings.ToLower(payload.ChatID), + Time: ts.UTC(), + ID: strings.ToLower(payload.ID), + }, nil +} diff --git a/internal/pgstore/store.go b/internal/pgstore/store.go new file mode 100644 index 0000000..da9def3 --- /dev/null +++ b/internal/pgstore/store.go @@ -0,0 +1,357 @@ +package pgstore + +import ( + "context" + "errors" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "mtusi.local/messenger/internal/model" +) + +type Store struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Ping(ctx context.Context) error { + if err := s.pool.Ping(ctx); err != nil { + return model.ErrUnavailable + } + return nil +} + +func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (model.User, error) { + if !model.ValidUsername(username) || !model.ValidLabel(displayName, 100) || passwordHash == "" { + return model.User{}, model.ErrInvalid + } + id, err := model.NewID() + if err != nil { + return model.User{}, err + } + created := time.Now().UTC().Truncate(time.Microsecond) + user := model.User{ID: id, Username: username, DisplayName: displayName, CreatedAt: created} + _, err = s.pool.Exec(ctx, ` + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1::uuid, $2, $3, $4, $5)`, + user.ID, user.Username, user.DisplayName, passwordHash, user.CreatedAt) + if err != nil { + return model.User{}, classify(err) + } + return s.UserByID(ctx, user.ID) +} + +func (s *Store) UserByID(ctx context.Context, id string) (model.User, error) { + id, ok := model.NormalizeUUID(id) + if !ok { + return model.User{}, model.ErrNotFound + } + user, _, err := s.scanUser(ctx, ` + SELECT id::text, username, display_name, created_at, password_hash + FROM users WHERE id = $1::uuid`, id) + return user, err +} + +func (s *Store) UserByUsername(ctx context.Context, username string) (model.User, string, error) { + if !model.ValidUsername(username) { + return model.User{}, "", model.ErrNotFound + } + return s.scanUser(ctx, ` + SELECT id::text, username, display_name, created_at, password_hash + FROM users WHERE username = $1`, username) +} + +func (s *Store) CreateChat(ctx context.Context, caller, title string, memberIDs []string) (model.Chat, error) { + if !model.ValidLabel(title, 100) || len(memberIDs) < 1 || len(memberIDs) > 100 { + return model.Chat{}, model.ErrInvalid + } + caller, ok := model.NormalizeUUID(caller) + if !ok { + return model.Chat{}, model.ErrInvalid + } + seen := make(map[string]struct{}, len(memberIDs)) + members := make([]string, 0, len(memberIDs)) + callerSeen := false + for _, raw := range memberIDs { + id, ok := model.NormalizeUUID(raw) + if !ok { + return model.Chat{}, model.ErrInvalid + } + if _, dup := seen[id]; dup { + return model.Chat{}, model.ErrInvalid + } + seen[id] = struct{}{} + if id == caller { + callerSeen = true + } + members = append(members, id) + } + if !callerSeen { + return model.Chat{}, model.ErrInvalid + } + id, err := model.NewID() + if err != nil { + return model.Chat{}, err + } + created := time.Now().UTC().Truncate(time.Microsecond) + tx, err := s.pool.Begin(ctx) + if err != nil { + return model.Chat{}, classify(err) + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, `INSERT INTO chats (id, title, created_at) VALUES ($1::uuid, $2, $3)`, id, title, created); err != nil { + return model.Chat{}, classify(err) + } + tag, err := tx.Exec(ctx, ` + INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1::uuid, ord.id::uuid, ord.pos::int - 1 + FROM unnest($2::text[]) WITH ORDINALITY AS ord(id, pos) + JOIN users AS u ON u.id = ord.id::uuid`, id, members) + if err != nil { + return model.Chat{}, classify(err) + } + if int(tag.RowsAffected()) != len(members) { + return model.Chat{}, model.ErrInvalid + } + if err := tx.Commit(ctx); err != nil { + return model.Chat{}, classify(err) + } + return model.Chat{ID: id, Title: title, MemberIDs: members, CreatedAt: created}, nil +} + +func (s *Store) Chat(ctx context.Context, chatID, userID string) (model.Chat, error) { + chatID, ok := model.NormalizeUUID(chatID) + if !ok { + return model.Chat{}, model.ErrInvalid + } + userID, ok = model.NormalizeUUID(userID) + if !ok { + return model.Chat{}, model.ErrInvalid + } + var chat model.Chat + err := s.pool.QueryRow(ctx, ` + SELECT c.id::text, c.title, c.created_at, + COALESCE(( + SELECT array_agg(cm.user_id::text ORDER BY cm.position) + FROM chat_members AS cm WHERE cm.chat_id = c.id + ), '{}'::text[]) + FROM chats AS c + JOIN chat_members AS mine ON mine.chat_id = c.id AND mine.user_id = $2::uuid + WHERE c.id = $1::uuid`, chatID, userID).Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs) + if err != nil { + return model.Chat{}, classify(err) + } + chat.CreatedAt = chat.CreatedAt.UTC() + if chat.MemberIDs == nil { + chat.MemberIDs = []string{} + } + return chat, nil +} + +func (s *Store) ListChats(ctx context.Context, userID string, limit int, cursor *model.Cursor) ([]model.Chat, *string, error) { + userID, ok := model.NormalizeUUID(userID) + if !ok { + return nil, nil, model.ErrInvalid + } + if cursor != nil && cursor.Resource != model.ResourceChats { + return nil, nil, model.ErrInvalid + } + var afterTime *time.Time + var afterID *string + if cursor != nil { + t := cursor.Time + id := cursor.ID + afterTime = &t + afterID = &id + } + rows, err := s.pool.Query(ctx, ` + SELECT c.id::text, c.title, c.created_at, + COALESCE(( + SELECT array_agg(cm.user_id::text ORDER BY cm.position) + FROM chat_members AS cm WHERE cm.chat_id = c.id + ), '{}'::text[]) + FROM chats AS c + JOIN chat_members AS mine ON mine.chat_id = c.id AND mine.user_id = $1::uuid + WHERE $2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid) + ORDER BY c.created_at, c.id + LIMIT $4`, userID, afterTime, afterID, limit+1) + if err != nil { + return nil, nil, classify(err) + } + defer rows.Close() + items := make([]model.Chat, 0) + for rows.Next() { + var chat model.Chat + if err := rows.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs); err != nil { + return nil, nil, classify(err) + } + chat.CreatedAt = chat.CreatedAt.UTC() + if chat.MemberIDs == nil { + chat.MemberIDs = []string{} + } + items = append(items, chat) + } + if err := rows.Err(); err != nil { + return nil, nil, classify(err) + } + return pageChats(items, limit) +} + +func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string) (model.Message, error) { + if !model.ValidLabel(text, 4000) { + return model.Message{}, model.ErrInvalid + } + chatID, ok := model.NormalizeUUID(chatID) + if !ok { + return model.Message{}, model.ErrInvalid + } + senderID, ok = model.NormalizeUUID(senderID) + if !ok { + return model.Message{}, model.ErrInvalid + } + id, err := model.NewID() + if err != nil { + return model.Message{}, err + } + created := time.Now().UTC().Truncate(time.Microsecond) + var msg model.Message + err = s.pool.QueryRow(ctx, ` + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT $1::uuid, $2::uuid, $3::uuid, $4, $5 + WHERE EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = $2::uuid AND user_id = $3::uuid + ) + RETURNING id::text, chat_id::text, sender_id::text, text, created_at`, + id, chatID, senderID, text, created).Scan(&msg.ID, &msg.ChatID, &msg.SenderID, &msg.Text, &msg.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return model.Message{}, model.ErrNotFound + } + if err != nil { + return model.Message{}, classify(err) + } + msg.CreatedAt = msg.CreatedAt.UTC() + return msg, nil +} + +func (s *Store) ListMessages(ctx context.Context, chatID, userID string, limit int, cursor *model.Cursor) ([]model.Message, *string, error) { + chatID, ok := model.NormalizeUUID(chatID) + if !ok { + return nil, nil, model.ErrInvalid + } + userID, ok = model.NormalizeUUID(userID) + if !ok { + return nil, nil, model.ErrInvalid + } + if cursor != nil && (cursor.Resource != model.ResourceMessages || cursor.ChatID != chatID) { + return nil, nil, model.ErrInvalid + } + if _, err := s.Chat(ctx, chatID, userID); err != nil { + return nil, nil, err + } + var afterTime *time.Time + var afterID *string + if cursor != nil { + t := cursor.Time + id := cursor.ID + afterTime = &t + afterID = &id + } + rows, err := s.pool.Query(ctx, ` + SELECT id::text, chat_id::text, sender_id::text, text, created_at + FROM messages + WHERE chat_id = $1::uuid + AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3::uuid)) + ORDER BY created_at, id + LIMIT $4`, chatID, afterTime, afterID, limit+1) + if err != nil { + return nil, nil, classify(err) + } + defer rows.Close() + items := make([]model.Message, 0) + for rows.Next() { + var msg model.Message + if err := rows.Scan(&msg.ID, &msg.ChatID, &msg.SenderID, &msg.Text, &msg.CreatedAt); err != nil { + return nil, nil, classify(err) + } + msg.CreatedAt = msg.CreatedAt.UTC() + items = append(items, msg) + } + if err := rows.Err(); err != nil { + return nil, nil, classify(err) + } + return pageMessages(items, limit, chatID) +} + +func (s *Store) scanUser(ctx context.Context, query string, arg any) (model.User, string, error) { + var user model.User + var hash string + err := s.pool.QueryRow(ctx, query, arg).Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt, &hash) + if err != nil { + return model.User{}, "", classify(err) + } + user.CreatedAt = user.CreatedAt.UTC() + return user, hash, nil +} + +func pageMessages(items []model.Message, limit int, chatID string) ([]model.Message, *string, error) { + if len(items) <= limit { + return items, nil, nil + } + items = items[:limit] + last := items[len(items)-1] + token, err := model.EncodeCursor(model.Cursor{ + Resource: model.ResourceMessages, + ChatID: chatID, + Time: last.CreatedAt, + ID: last.ID, + }) + if err != nil { + return nil, nil, err + } + return items, &token, nil +} + +func pageChats(items []model.Chat, limit int) ([]model.Chat, *string, error) { + if len(items) <= limit { + return items, nil, nil + } + items = items[:limit] + last := items[len(items)-1] + token, err := model.EncodeCursor(model.Cursor{ + Resource: model.ResourceChats, + Time: last.CreatedAt, + ID: last.ID, + }) + if err != nil { + return nil, nil, err + } + return items, &token, nil +} + +func classify(err error) error { + if err == nil || errors.Is(err, model.ErrInvalid) || errors.Is(err, model.ErrConflict) || errors.Is(err, model.ErrNotFound) || errors.Is(err, model.ErrUnavailable) { + return err + } + if errors.Is(err, pgx.ErrNoRows) { + return model.ErrNotFound + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "23505": + return model.ErrConflict + case "23503", "23514", "23502", "22P02": + return model.ErrInvalid + default: + return model.ErrUnavailable + } + } + return model.ErrUnavailable +} diff --git a/internal/sessions/redis.go b/internal/sessions/redis.go new file mode 100644 index 0000000..36dba02 --- /dev/null +++ b/internal/sessions/redis.go @@ -0,0 +1,85 @@ +package sessions + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "time" + + "github.com/redis/go-redis/v9" + + "mtusi.local/messenger/internal/model" +) + +type Store struct { + client *redis.Client + ttl time.Duration +} + +type record struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +func New(client *redis.Client, ttl time.Duration) *Store { + return &Store{client: client, ttl: ttl} +} + +func (s *Store) Ping(ctx context.Context) error { + if err := s.client.Ping(ctx).Err(); err != nil { + return model.ErrUnavailable + } + return nil +} + +func (s *Store) Create(ctx context.Context, userID string) (string, time.Time, error) { + var raw [32]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", time.Time{}, err + } + token := base64.RawURLEncoding.EncodeToString(raw[:]) + expires := time.Now().UTC().Add(s.ttl).Truncate(time.Microsecond) + body, err := json.Marshal(record{UserID: userID, ExpiresAt: expires}) + if err != nil { + return "", time.Time{}, err + } + if err := s.client.Set(ctx, key(token), body, s.ttl).Err(); err != nil { + return "", time.Time{}, model.ErrUnavailable + } + return token, expires, nil +} + +func (s *Store) Lookup(ctx context.Context, token string) (string, error) { + body, err := s.client.Get(ctx, key(token)).Bytes() + if errors.Is(err, redis.Nil) { + return "", model.ErrNotFound + } + if err != nil { + return "", model.ErrUnavailable + } + var item record + if err := json.Unmarshal(body, &item); err != nil || item.UserID == "" { + return "", model.ErrNotFound + } + if !item.ExpiresAt.After(time.Now()) { + _ = s.client.Del(ctx, key(token)).Err() + return "", model.ErrNotFound + } + return item.UserID, nil +} + +func (s *Store) Revoke(ctx context.Context, token string) error { + if err := s.client.Del(ctx, key(token)).Err(); err != nil { + return model.ErrUnavailable + } + return nil +} + +func key(token string) string { + sum := sha256.Sum256([]byte(token)) + return "sess:" + hex.EncodeToString(sum[:]) +} diff --git a/migrations/001_tables.sql b/migrations/001_tables.sql new file mode 100644 index 0000000..e6c3560 --- /dev/null +++ b/migrations/001_tables.sql @@ -0,0 +1,38 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE INDEX chats_order_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_pos CHECK (position >= 0) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); diff --git a/migrations/002_message_history_index.sql b/migrations/002_message_history_index.sql new file mode 100644 index 0000000..e31c7b1 --- /dev/null +++ b/migrations/002_message_history_index.sql @@ -0,0 +1 @@ +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/postman/lab2.postman_collection.json b/postman/lab2.postman_collection.json new file mode 100644 index 0000000..bede8c3 --- /dev/null +++ b/postman/lab2.postman_collection.json @@ -0,0 +1,211 @@ +{ + "info": { + "name": "MTUSI lab2 messenger", + "description": "Local stand at http://localhost:8080. Run the folder from top to bottom. The first request fills usernames. Passwords stay in collection variables and are the synthetic lab password.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + {"key": "baseUrl", "value": "http://localhost:8080"}, + {"key": "password", "value": "Example-only-not-a-real-secret-123"}, + {"key": "aliceUsername", "value": ""}, + {"key": "bobUsername", "value": ""}, + {"key": "aliceId", "value": ""}, + {"key": "bobId", "value": ""}, + {"key": "token", "value": ""}, + {"key": "chatId", "value": ""}, + {"key": "cursor", "value": ""} + ], + "item": [ + { + "name": "01 health live", + "request": {"method": "GET", "header": [], "url": "{{baseUrl}}/health/live"} + }, + { + "name": "02 health ready", + "request": {"method": "GET", "header": [], "url": "{{baseUrl}}/health/ready"} + }, + { + "name": "03 register Alice", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "const stamp = Date.now().toString(36);", + "pm.collectionVariables.set('aliceUsername', ('alice_' + stamp).slice(0, 32));", + "pm.collectionVariables.set('bobUsername', ('bob_' + stamp).slice(0, 32));" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const body = pm.response.json();", + "pm.collectionVariables.set('aliceId', body.id);" + ] + } + } + ], + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": "{\n \"username\": \"{{aliceUsername}}\",\n \"display_name\": \"Алиса\",\n \"password\": \"{{password}}\"\n}" + }, + "url": "{{baseUrl}}/api/v1/users" + } + }, + { + "name": "04 register Bob", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": ["pm.collectionVariables.set('bobId', pm.response.json().id);"] + } + } + ], + "request": { + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": "{\n \"username\": \"{{bobUsername}}\",\n \"display_name\": \"Боб\",\n \"password\": \"{{password}}\"\n}" + }, + "url": "{{baseUrl}}/api/v1/users" + } + }, + { + "name": "05 login Alice Basic", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": ["pm.collectionVariables.set('token', pm.response.json().token);"] + } + } + ], + "request": { + "auth": { + "type": "basic", + "basic": [ + {"key": "username", "value": "{{aliceUsername}}", "type": "string"}, + {"key": "password", "value": "{{password}}", "type": "string"} + ] + }, + "method": "POST", + "header": [], + "url": "{{baseUrl}}/api/v1/auth/sessions" + } + }, + { + "name": "06 me with Bearer", + "request": { + "auth": {"type": "bearer", "bearer": [{"key": "token", "value": "{{token}}", "type": "string"}]}, + "method": "GET", + "header": [], + "url": "{{baseUrl}}/api/v1/users/me" + } + }, + { + "name": "07 me with only X-User-Id", + "request": { + "method": "GET", + "header": [{"key": "X-User-Id", "value": "{{aliceId}}"}], + "url": "{{baseUrl}}/api/v1/users/me" + } + }, + { + "name": "08 wrong password", + "request": { + "auth": { + "type": "basic", + "basic": [ + {"key": "username", "value": "{{aliceUsername}}", "type": "string"}, + {"key": "password", "value": "wrong-password-value", "type": "string"} + ] + }, + "method": "POST", + "header": [], + "url": "{{baseUrl}}/api/v1/auth/sessions" + } + }, + { + "name": "09 create chat", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": ["pm.collectionVariables.set('chatId', pm.response.json().id);"] + } + } + ], + "request": { + "auth": {"type": "bearer", "bearer": [{"key": "token", "value": "{{token}}", "type": "string"}]}, + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": { + "mode": "raw", + "raw": "{\n \"title\": \"Проект по ВНП\",\n \"member_ids\": [\"{{aliceId}}\", \"{{bobId}}\"]\n}" + }, + "url": "{{baseUrl}}/api/v1/chats" + } + }, + { + "name": "10 send message", + "request": { + "auth": {"type": "bearer", "bearer": [{"key": "token", "value": "{{token}}", "type": "string"}]}, + "method": "POST", + "header": [{"key": "Content-Type", "value": "application/json"}], + "body": {"mode": "raw", "raw": "{\n \"text\": \"Привет, Боб!\"\n}"}, + "url": "{{baseUrl}}/api/v1/chats/{{chatId}}/messages" + } + }, + { + "name": "11 first history page", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const cursor = pm.response.json().next_cursor;", + "if (cursor) pm.collectionVariables.set('cursor', cursor);" + ] + } + } + ], + "request": { + "auth": {"type": "bearer", "bearer": [{"key": "token", "value": "{{token}}", "type": "string"}]}, + "method": "GET", + "header": [], + "url": "{{baseUrl}}/api/v1/chats/{{chatId}}/messages?limit=50" + } + }, + { + "name": "12 logout", + "request": { + "auth": {"type": "bearer", "bearer": [{"key": "token", "value": "{{token}}", "type": "string"}]}, + "method": "DELETE", + "header": [], + "url": "{{baseUrl}}/api/v1/auth/sessions/current" + } + }, + { + "name": "13 me after logout", + "request": { + "auth": {"type": "bearer", "bearer": [{"key": "token", "value": "{{token}}", "type": "string"}]}, + "method": "GET", + "header": [], + "url": "{{baseUrl}}/api/v1/users/me" + } + } + ] +} diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..d3020a8 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart api postgres redis diff --git a/student-tests/deps.py b/student-tests/deps.py new file mode 100644 index 0000000..5f649a9 --- /dev/null +++ b/student-tests/deps.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Stop Redis, then Postgres, and check the API does not trust an anonymous caller.""" +import subprocess +import sys +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, arguments, check, run + + +def compose(*args): + completed = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + text=True, + capture_output=True, + ) + check(completed.returncode == 0, "compose " + " ".join(args) + " failed") + + +def wait_ready(client, timeout): + deadline = time.monotonic() + timeout + while True: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + check(time.monotonic() < deadline, "API did not become ready") + time.sleep(0.5) + + +def main(): + args = arguments("Dependency failure checks") + client = Client(args.base_url, args.ca_file, timeout=8) + wait_ready(client, 30) + actor = client.user("deps") + try: + compose("stop", "redis") + time.sleep(1) + ready = client.call("GET", "/health/ready", 503) + check(ready["json"]["error"]["code"] == "unavailable", "ready code") + anon = client.call("GET", "/api/v1/users/me", 401) + check(anon["json"]["error"]["code"] == "unauthorized", "anonymous became trusted while redis was down") + authed = client.call("GET", "/api/v1/users/me", 503, actor=actor) + check(authed["json"]["error"]["code"] == "unavailable", "bearer during redis outage") + check("labredispass" not in authed["json"]["error"]["message"], "secret leaked") + finally: + compose("start", "redis") + wait_ready(client, 40) + client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + try: + compose("stop", "postgres") + time.sleep(1) + client.call("GET", "/health/ready", 503) + client.call("GET", "/api/v1/users/me", 401) + client.call( + "POST", + "/api/v1/users", + 503, + body={"username": "down_" + uuid.uuid4().hex[:8], "display_name": "Down", "password": "long-enough-password"}, + ) + finally: + compose("start", "postgres") + wait_ready(client, 40) + print("PASS: dependency failures stay untrusted") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/expiry.py b/student-tests/expiry.py new file mode 100644 index 0000000..9a9e858 --- /dev/null +++ b/student-tests/expiry.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Login against an API started with a short SESSION_TTL_SECONDS and wait it out.""" +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, arguments, check, run + + +def main(): + def configure(parser): + parser.add_argument("--wait", type=float, default=3) + + args = arguments("Session expiry", configure) + client = Client(args.base_url, args.ca_file, timeout=15) + actor = client.user("ttl") + client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + time.sleep(args.wait) + client.call("GET", "/api/v1/users/me", 401, actor=actor) + check(True, "") + print("PASS: session expired after %.1f s" % args.wait) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 0000000..4018416 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Grade-4 checks that the public smoke does not cover.""" +import base64 +import sys +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, arguments, check, run + + +def walk(client, actor, chat, limit): + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=%d" % limit + items = [] + cursor = None + guard = 0 + while True: + guard += 1 + check(guard < 10000, "pagination did not finish") + target = path if cursor is None else path + "&cursor=" + quote(cursor, safe="") + page = client.call("GET", target, actor=actor, schema="MessagePage")["json"] + check(len(page["items"]) <= limit, "page is larger than limit") + items.extend(page["items"]) + if page["next_cursor"] is None: + return items + check(page["items"], "cursor was returned for an empty page") + cursor = page["next_cursor"] + + +def test_input(client): + client.call("POST", "/api/v1/users", 400, raw=b"{", headers={"Content-Type": "application/json"}) + client.call( + "POST", + "/api/v1/users", + 400, + raw=b'{"username":"shortpwuser","display_name":"Name","password":"short"}', + headers={"Content-Type": "application/json"}, + ) + client.call( + "POST", + "/api/v1/users", + 400, + raw=b'{"Username":"caseuserx","display_name":"Name","password":"long-enough-password"}', + headers={"Content-Type": "application/json"}, + ) + client.call( + "POST", + "/api/v1/users", + 400, + raw=b'{"username":"utf8userx","display_name":"a\xff","password":"long-enough-password"}', + headers={"Content-Type": "application/json"}, + ) + client.call( + "POST", + "/api/v1/users", + 400, + raw=b'{"username":"extra_userx","display_name":"Name","password":"long-enough-password","role":"admin"}', + headers={"Content-Type": "application/json"}, + ) + + +def test_duplicate_and_race(client): + username = "race_" + uuid.uuid4().hex[:8] + body = {"username": username, "display_name": "Race", "password": "long-enough-password"} + client.call("POST", "/api/v1/users", 201, body=body, schema="User") + conflict = client.call("POST", "/api/v1/users", 409, body=body) + check(conflict["json"]["error"]["code"] == "conflict", "duplicate username code") + + shared = "par_" + uuid.uuid4().hex[:8] + payload = {"username": shared, "display_name": "Parallel", "password": "long-enough-password"} + + def once(_): + return client.call("POST", "/api/v1/users", [201, 409], body=payload)["status"] + + with ThreadPoolExecutor(max_workers=8) as pool: + codes = list(pool.map(once, range(8))) + check(codes.count(201) == 1, "parallel registration created %s users" % codes.count(201)) + check(all(code in (201, 409) for code in codes), "parallel registration returned %s" % codes) + + +def test_auth(client): + alice = client.user("auth") + bad_user = client.call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + base64.b64encode(b"missing_user_name:long-enough-password").decode()}, + ) + bad_pass = client.call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + base64.b64encode( + (alice["registration"]["username"] + ":wrong-password-value").encode() + ).decode()}, + ) + check(bad_user["json"]["error"]["message"] == bad_pass["json"]["error"]["message"], "credential errors differ") + client.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + me = client.call( + "GET", + "/api/v1/users/me", + actor=alice, + headers={"X-User-Id": "00000000-0000-4000-8000-000000000001"}, + schema="User", + )["json"] + check(me["id"] == alice["id"], "X-User-Id changed the session user") + client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + client.call("GET", "/api/v1/users/me", 401, actor=alice) + + +def test_chat_rollback_and_history(client): + alice = client.user("chat") + bob = client.user("peer") + missing = str(uuid.uuid4()) + client.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + body={"title": "broken", "member_ids": [alice["id"], missing]}, + ) + listed = client.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"] + check(listed["items"] == [], "rolled back chat is visible") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "ok", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + eve = client.user("outsider") + client.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve) + client.call("POST", "/api/v1/chats/" + chat["id"] + "/messages", 400, actor=alice, body={"text": " "}) + for n in range(25): + client.message(alice, chat, "m%02d" % n) + items = walk(client, bob, chat, 10) + check(len(items) == 25, "history length") + check(len({item["id"] for item in items}) == 25, "duplicate ids") + check([item["text"] for item in items] == ["m%02d" % n for n in range(25)], "history order") + foreign = walk(client, bob, chat, 10) + cursor = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=1", + actor=alice, + schema="MessagePage", + )["json"]["next_cursor"] + other = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "second", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + client.call( + "GET", + "/api/v1/chats/" + other["id"] + "/messages?limit=10&cursor=" + quote(cursor, safe=""), + 400, + actor=alice, + ) + check(len(foreign) == 25, "second walk") + + +def main(): + args = arguments("Student checks for lab2") + client = Client(args.base_url, args.ca_file, timeout=20) + client.call("GET", "/health/ready", schema="Health") + test_input(client) + test_duplicate_and_race(client) + test_auth(client) + test_chat_rollback_and_history(client) + print("PASS: student checks") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/track2.py b/student-tests/track2.py new file mode 100644 index 0000000..5dd9c84 --- /dev/null +++ b/student-tests/track2.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Track 2: two history plans, write cost, backup into a second Postgres volume.""" +import json +import os +import platform +import re +import subprocess +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" + + +def env_value(name): + for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1].strip() + raise SystemExit("missing " + name) + + +def run(args, input_bytes=None, check=True): + completed = subprocess.run( + args, + cwd=ROOT, + input=input_bytes, + capture_output=True, + ) + if check and completed.returncode != 0: + raise SystemExit(completed.stderr.decode("utf-8", "replace")[-2000:]) + return completed + + +def compose(args, input_bytes=None, check=True): + return run(["docker", "compose", "--env-file", ".env", *args], input_bytes, check) + + +def psql(service, database, sql, check=True, tuples=False): + command = ["exec", "-T", service, "psql", "-v", "ON_ERROR_STOP=1", "-X", "-q", + "-U", env_value("POSTGRES_USER"), "-d", database] + if tuples: + command.extend(["-A", "-t", "-F", "|"]) + completed = compose(command, sql.encode("utf-8"), check) + text = completed.stdout.decode("utf-8", "replace") + err = completed.stderr.decode("utf-8", "replace") + return text, err + + +def psql_rows(service, database, sql): + text, _ = psql(service, database, sql, tuples=True) + rows = [] + for line in text.splitlines(): + line = line.strip() + if line: + rows.append(line.split("|")) + return rows + + +def wait_service(service): + for _ in range(40): + probe = compose( + ["exec", "-T", service, "pg_isready", "-U", env_value("POSTGRES_USER"), "-d", env_value("POSTGRES_DB")], + check=False, + ) + if probe.returncode == 0: + return + time.sleep(1) + raise SystemExit(service + " did not become ready") + + +def recreate(service): + user = env_value("POSTGRES_USER") + base = env_value("POSTGRES_DB") + psql(service, base, "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'track2' AND pid <> pg_backend_pid();", check=False) + psql(service, base, "DROP DATABASE IF EXISTS track2;") + psql(service, base, "CREATE DATABASE track2;") + + +def apply_schema(): + for name in ("001_tables.sql", "002_message_history_index.sql"): + body = (ROOT / "migrations" / name).read_bytes() + psql("postgres", "track2", body.decode("utf-8")) + + +def seed(): + alice = "11111111-1111-4111-8111-111111111111" + bob = "22222222-2222-4222-8222-222222222222" + sizes = [70000, 20000, 8000, 1500, 400, 80, 15, 5] + [1] * 40 + chats = [] + statements = [ + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) VALUES + ('%s', 'track_alice', 'Алиса', 'not-a-login-hash', timestamptz '2026-01-01+00'), + ('%s', 'track_bob', 'Боб', 'not-a-login-hash', timestamptz '2026-01-01+00'); + """ % (alice, bob) + ] + for size in sizes: + chat_id = str(uuid.uuid4()) + chats.append({"id": chat_id, "messages": size}) + statements.append( + "INSERT INTO chats (id, title, created_at) VALUES ('%s', 'chat-%d', timestamptz '2026-01-01+00');" % (chat_id, size) + ) + statements.append( + """ + INSERT INTO chat_members (chat_id, user_id, position) VALUES + ('%s', '%s', 0), + ('%s', '%s', 1); + """ % (chat_id, alice, chat_id, bob) + ) + if size: + statements.append( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), '%s'::uuid, '%s'::uuid, 'm' || g, + timestamptz '2026-01-01+00' + (g * interval '1 millisecond') + FROM generate_series(1, %d) AS g; + """ % (chat_id, alice, size) + ) + for _ in range(25): + chat_id = str(uuid.uuid4()) + chats.append({"id": chat_id, "messages": 0}) + statements.append( + "INSERT INTO chats (id, title, created_at) VALUES ('%s', 'empty', timestamptz '2026-01-01+00');" % chat_id + ) + statements.append( + """ + INSERT INTO chat_members (chat_id, user_id, position) VALUES + ('%s', '%s', 0), + ('%s', '%s', 1); + """ % (chat_id, alice, chat_id, bob) + ) + psql("postgres", "track2", "\n".join(statements)) + psql("postgres", "track2", "ANALYZE users, chats, chat_members, messages;") + return alice, chats + + +def explain(sql): + text, _ = psql("postgres", "track2", "EXPLAIN (ANALYZE, BUFFERS)\n" + sql) + return text + + +def metric(plan): + execution = re.search(r"Execution Time: ([0-9.]+) ms", plan) + buffers = re.search(r"^Buffers: (.+)$", plan, re.M) + shared_hit = re.search(r"shared hit=(\d+)", plan) + shared_read = re.search(r"shared read=(\d+)", plan) + return { + "execution_ms": float(execution.group(1)) if execution else None, + "buffers": buffers.group(1).strip() if buffers else "", + "shared_hit": int(shared_hit.group(1)) if shared_hit else 0, + "shared_read": int(shared_read.group(1)) if shared_read else 0, + "uses_history_index": "messages_history_idx" in plan, + "node": plan_node(plan), + } + + +def plan_node(plan): + if "Index Only Scan" in plan: + return "Index Only Scan" + if "Bitmap" in plan: + return "Bitmap" + if "Index Scan" in plan: + return "Index Scan" + if "Seq Scan" in plan: + return "Seq Scan" + return "other" + + +def keyset_sql(table, chat_id, created_at, message_id, limit): + return """ + SELECT id::text, chat_id::text, sender_id::text, text, created_at + FROM %s + WHERE chat_id = '%s'::uuid + AND (created_at, id) > (timestamptz '%s+00', '%s'::uuid) + ORDER BY created_at, id + LIMIT %d + """ % (table, chat_id, created_at, message_id, limit) + + +def cursor_at(chat_id, offset): + rows = psql_rows("postgres", "track2", """ + SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), id::text + FROM messages + WHERE chat_id = '%s'::uuid + ORDER BY created_at, id + OFFSET %d LIMIT 1; + """ % (chat_id, offset)) + if not rows: + raise SystemExit("cursor row missing") + return rows[0][0].strip(), rows[0][1].strip() + + +def write_cost(chat_id, sender_id): + sql = """ + CREATE TABLE messages_pk ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL + ); + CREATE TABLE messages_idx ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL + ); + CREATE INDEX messages_idx_history ON messages_idx (chat_id, created_at, id); + CREATE TEMP TABLE seed AS + SELECT gen_random_uuid() AS id, + '%s'::uuid AS chat_id, + '%s'::uuid AS sender_id, + 'w' || g AS text, + timestamptz '2026-03-01+00' + (g * interval '1 millisecond') AS created_at + FROM generate_series(1, 50000) AS g; + DO $$ + DECLARE started timestamptz; + BEGIN + started := clock_timestamp(); + INSERT INTO messages_pk SELECT * FROM seed; + RAISE NOTICE 'pk_ms=%%', round(extract(epoch FROM clock_timestamp() - started) * 1000); + started := clock_timestamp(); + INSERT INTO messages_idx SELECT * FROM seed; + RAISE NOTICE 'idx_ms=%%', round(extract(epoch FROM clock_timestamp() - started) * 1000); + END $$; + DROP TABLE messages_pk, messages_idx; + """ % (chat_id, sender_id) + _, err = psql("postgres", "track2", sql) + pk = re.search(r"pk_ms=([0-9.]+)", err) + idx = re.search(r"idx_ms=([0-9.]+)", err) + if not pk or not idx: + raise SystemExit("write timings missing:\n" + err[-1500:]) + return {"rows": 50000, "primary_key_ms": float(pk.group(1)), "primary_key_and_history_index_ms": float(idx.group(1))} + + +def counts(service): + rows = psql_rows(service, "track2", """ + SELECT 'users', count(*)::text FROM users + UNION ALL SELECT 'chats', count(*)::text FROM chats + UNION ALL SELECT 'chat_members', count(*)::text FROM chat_members + UNION ALL SELECT 'messages', count(*)::text FROM messages; + """) + return {name: int(value) for name, value in rows} + + +def sample(service, chat_id): + return psql_rows(service, "track2", """ + SELECT id::text || '|' || text || '|' || to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') + FROM messages + WHERE chat_id = '%s'::uuid + ORDER BY created_at, id + LIMIT 3; + """ % chat_id) + + +def memory_gib(): + if os.name != "nt": + return None + import ctypes + + class Status(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("ullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + status = Status() + status.dwLength = ctypes.sizeof(Status) + if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return None + return round(status.ullTotalPhys / 1024**3, 1) + + +def main(): + EVIDENCE.mkdir(exist_ok=True) + compose(["--profile", "restore", "up", "-d", "postgres-restore"]) + wait_service("postgres") + wait_service("postgres-restore") + recreate("postgres") + recreate("postgres-restore") + apply_schema() + sender, chats = seed() + populated = [item for item in chats if item["messages"] >= 10000] + big = max(chats, key=lambda item: item["messages"]) + medium = populated[-1] + created_at, message_id = cursor_at(big["id"], 60000) + plans = { + "keyset_deep_history_index": explain(keyset_sql("messages", big["id"], created_at, message_id, 50)), + "offset_deep_same_chat": explain(""" + SELECT id::text, chat_id::text, sender_id::text, text, created_at + FROM messages + WHERE chat_id = '%s'::uuid + ORDER BY created_at, id + OFFSET 60000 LIMIT 50 + """ % big["id"]), + "keyset_first_page_at_least_10000": explain(""" + SELECT id::text, chat_id::text, sender_id::text, text, created_at + FROM messages + WHERE chat_id = '%s'::uuid + ORDER BY created_at, id + LIMIT 50 + """ % medium["id"]), + } + psql("postgres", "track2", """ + CREATE TABLE messages_time AS TABLE messages; + CREATE INDEX messages_time_created ON messages_time (created_at); + ANALYZE messages_time; + """) + plans["keyset_deep_created_at_index_only"] = explain(keyset_sql("messages_time", big["id"], created_at, message_id, 50)) + psql("postgres", "track2", "DROP TABLE messages_time;") + writes = write_cost(big["id"], sender) + before = counts("postgres") + chosen = sample("postgres", big["id"]) + settings = psql_rows("postgres", "track2", """ + SELECT current_setting('server_version') + UNION ALL SELECT current_setting('synchronous_commit') + UNION ALL SELECT current_setting('fsync'); + """) + flat_settings = [row[0] for row in settings] + redis_cfg = compose( + ["exec", "-T", "redis", "sh", "-c", + 'redis-cli --no-auth-warning -a "$REDIS_PASSWORD" CONFIG GET appendonly; redis-cli --no-auth-warning -a "$REDIS_PASSWORD" CONFIG GET appendfsync'] + ).stdout.decode("utf-8", "replace") + started = time.perf_counter() + dump = compose(["exec", "-T", "postgres", "pg_dump", "-U", env_value("POSTGRES_USER"), "-d", "track2", "-Fc", "--no-owner", "--no-acl"]).stdout + dump_seconds = round(time.perf_counter() - started, 3) + extra_id = str(uuid.uuid4()) + psql("postgres", "track2", """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ('%s', '%s', '%s', 'after-dump', timestamptz '2026-06-01+00'); + """ % (extra_id, big["id"], sender)) + started = time.perf_counter() + restored = compose( + ["exec", "-T", "postgres-restore", "pg_restore", "-U", env_value("POSTGRES_USER"), "-d", "track2", "--no-owner", "--exit-on-error"], + dump, + ) + restore_seconds = round(time.perf_counter() - started, 3) + if restored.returncode != 0: + raise SystemExit(restored.stderr.decode("utf-8", "replace")[-2000:]) + after_source = counts("postgres") + after_restore = counts("postgres-restore") + restored_sample = sample("postgres-restore", big["id"]) + extra_in_restore = psql_rows("postgres-restore", "track2", "SELECT count(*)::text FROM messages WHERE id = '%s'::uuid;" % extra_id) + summary = { + "track": 2, + "hypothesis": "Keyset on (chat_id, created_at, id) reads about one page even deep in a large chat. OFFSET on the same index walks the skipped prefix. An index on created_at alone is also selective at a timestamp that only the largest chat has reached, so it is not a substitute for bounding the walk by chat_id. The extra history index increases insert time. A custom-format dump restored into a second Postgres volume matches the snapshot and omits a row committed after pg_dump started.", + "hardware": { + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "ram_gib": memory_gib(), + "api_limit": "2 CPU, 768 MiB", + }, + "postgres": { + "image": "postgres:16.10-alpine", + "version": flat_settings[0] if flat_settings else "", + "synchronous_commit": flat_settings[1] if len(flat_settings) > 1 else "", + "fsync": flat_settings[2] if len(flat_settings) > 2 else "", + }, + "redis_persistence": redis_cfg.strip(), + "distribution": { + "messages": before["messages"], + "chats": before["chats"], + "largest_chat": big["messages"], + "empty_chats": sum(1 for item in chats if item["messages"] == 0), + "sizes": sorted({item["messages"] for item in chats}, reverse=True), + }, + "queries": {name: metric(plan) for name, plan in plans.items()}, + "write": writes, + "backup": { + "dump_seconds": dump_seconds, + "restore_seconds": restore_seconds, + "dump_bytes": len(dump), + "counts_before_dump": before, + "counts_source_after_extra_commit": after_source, + "counts_restored": after_restore, + "sample_match": chosen == restored_sample, + "extra_row_in_restore": int(extra_in_restore[0][0]) if extra_in_restore else None, + }, + } + (EVIDENCE / "track2.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + chunks = [] + for name, plan in plans.items(): + chunks.append("===== " + name + " =====\n" + plan.strip() + "\n") + (EVIDENCE / "track2-explain.txt").write_text("\n".join(chunks), encoding="utf-8") + print(json.dumps({"messages": before["messages"], "queries": summary["queries"], "write": writes, "backup": summary["backup"]}, ensure_ascii=False, indent=2)) + if not summary["backup"]["sample_match"]: + raise SystemExit("restored sample differs") + if summary["backup"]["extra_row_in_restore"] != 0: + raise SystemExit("row committed after the dump was restored") + if before["messages"] < 100000: + raise SystemExit("not enough messages") + print("PASS: track2") + + +if __name__ == "__main__": + main()