From a434e35336d217cb95d100b771e59c734fe36989 Mon Sep 17 00:00:00 2001 From: R0flyan Date: Fri, 16 Oct 2026 15:24:08 +0300 Subject: [PATCH] lab 2 realization: PostgreSQL, Valkey, track 2 --- .dockerignore | 27 +- .env.example | 10 +- .gitattributes | 3 + .github/pull_request_template.md | 58 +++-- Dockerfile | 15 ++ Dockerfile.tests | 7 + Makefile | 18 ++ REPORT.md | 199 +++++++++++++-- auth.go | 147 +++++++++++ auth_test.go | 50 ++++ compose.yaml | 62 +++++ evidence/audit.json | 19 ++ evidence/check.txt | 5 + evidence/clean-build.txt | 147 +++++++++++ evidence/explain-run.txt | 6 + evidence/explain.txt | 32 +++ evidence/infrastructure.json | 55 +++++ evidence/infrastructure.txt | 10 + evidence/linux-test-persistence.txt | 6 + evidence/own.txt | 29 +++ evidence/race.txt | 16 ++ evidence/research-plans.txt | 129 ++++++++++ evidence/research.csv | 367 ++++++++++++++++++++++++++++ evidence/research.json | 199 +++++++++++++++ evidence/restart-after.txt | 3 + evidence/restart-before.txt | 3 + evidence/smoke.txt | 5 + evidence/source-check.txt | 4 + evidence/source-manifest.json | 74 ++++++ go.mod | 20 ++ go.sum | 40 +++ main.go | 360 +++++++++++++++++++++++++++ migrations/001_initial.sql | 28 +++ migrations/002_unicode_storage.sql | 12 + protocol.go | 237 ++++++++++++++++++ scripts/bootstrap.py | 29 +++ scripts/linux_tests.py | 17 ++ scripts/restart.sh | 19 ++ store.go | 223 +++++++++++++++++ student-tests/README.md | 118 +++++++++ student-tests/acceptance.py | 161 ++++++++++++ student-tests/authentication.py | 70 ++++++ student-tests/explain.py | 40 +++ student-tests/infrastructure.py | 134 ++++++++++ student-tests/research.py | 227 +++++++++++++++++ 45 files changed, 3386 insertions(+), 54 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 Dockerfile.tests create mode 100644 auth.go create mode 100644 auth_test.go create mode 100644 compose.yaml create mode 100644 evidence/audit.json create mode 100644 evidence/check.txt create mode 100644 evidence/clean-build.txt create mode 100644 evidence/explain-run.txt create mode 100644 evidence/explain.txt create mode 100644 evidence/infrastructure.json create mode 100644 evidence/infrastructure.txt create mode 100644 evidence/linux-test-persistence.txt create mode 100644 evidence/own.txt create mode 100644 evidence/race.txt create mode 100644 evidence/research-plans.txt create mode 100644 evidence/research.csv create mode 100644 evidence/research.json create mode 100644 evidence/restart-after.txt create mode 100644 evidence/restart-before.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/source-check.txt create mode 100644 evidence/source-manifest.json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_unicode_storage.sql create mode 100644 protocol.go create mode 100644 scripts/bootstrap.py create mode 100644 scripts/linux_tests.py create mode 100755 scripts/restart.sh create mode 100644 store.go create mode 100644 student-tests/README.md create mode 100644 student-tests/acceptance.py create mode 100644 student-tests/authentication.py create mode 100644 student-tests/explain.py create mode 100644 student-tests/infrastructure.py create mode 100644 student-tests/research.py diff --git a/.dockerignore b/.dockerignore index bc16e55..7dfceef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,7 @@ -.git -.github -.env -.env.* -!.env.example -secrets -certs -volumes -data -backups -*.key -*.pem -*.p12 -*.pfx -*.jks -.venv -__pycache__ -node_modules -evidence -.DS_Store +** +!go.mod +!go.sum +!*.go +!migrations/ +!migrations/*.sql +!Dockerfile diff --git a/.env.example b/.env.example index 714c807..612036d 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,11 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Local configuration; secrets are generated by scripts/bootstrap.py. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. +SESSION_STORE=valkey +DB_POOL_MAX=8 +MAX_INFLIGHT=32 DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url +CURSOR_SECRET_FILE=/run/secrets/cursor_secret diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2a4cfbf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.sh text eol=lf +*.go text eol=lf +*.sql text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8ceefa1..354dc0e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,48 @@ ## Лабораторная 2 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Максим Ситосенко / БВТ2303. +Целевая оценка (3/4/5): **5**. +Трек 5, если заявлен: **2 — план запроса и восстановление**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: API lab1 перенесён в PostgreSQL; добавлены +миграции, транзакции, Argon2id, Basic-вход и Bearer-сессии в Valkey с TTL/AOF. +Сохранены валидация, членство, пагинация и корректный SIGTERM. Добавлены Compose, +собственные тесты отказов и EXPLAIN. Исследованы два индекса на 100 000 +сообщений с неравномерными чатами, стоимость записи и восстановление backup +в новый volume со сверкой всей истории по SHA-256. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Команды запуска из чистого clone: + +```sh +python3 -X utf8 scripts/bootstrap.py +docker compose up -d --build --wait +make check +make test BASE_URL=http://127.0.0.1:8080 +chmod +x scripts/restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 +make test-own BASE_URL=http://127.0.0.1:8080 +make test-infra BASE_URL=http://127.0.0.1:8080 +make explain +make research +``` + +Результаты проверок и ссылка на REPORT.md: [отчёт](../REPORT.md), +реальный вывод и планы — в `evidence/`. Smoke и persistence пройдены; +гонка username: 1×201, 9×409; 100/100 сообщений; rollback, TTL, logout и отказы +хранилищ проверены. Исследование и restore прошли: совпали 100 000 сообщений, +11 выбранных записей и SHA-256 всей истории; запись после backup отсутствует +в восстановленной БД. Измерения и границы потери данных приведены в REPORT.md. + +Ограничения / известные ошибки: локальный HTTP, одна БД и один Valkey, +без HA, автоматического backup и PITR; cursor не гарантирует snapshot. +Эксперимент выполнен на тёплом кэше с одним клиентом, backup охватывает PostgreSQL. + +Источники и использование ИИ: PostgreSQL/Valkey/Argon2id — ссылки в REPORT.md; +Codex помог с реализацией, тестами и отчётом. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4812c65 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.26.0-bookworm@sha256:2a0ba12e116687098780d3ce700f9ce3cb340783779646aafbabed748fa6677c AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +COPY migrations ./migrations +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /messenger . +FROM scratch AS runtime +COPY --from=build /messenger /messenger +USER 65532:65532 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 GOMEMLIMIT=256MiB +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 CMD ["/messenger","healthcheck"] +STOPSIGNAL SIGTERM +ENTRYPOINT ["/messenger"] diff --git a/Dockerfile.tests b/Dockerfile.tests new file mode 100644 index 0000000..6b10d14 --- /dev/null +++ b/Dockerfile.tests @@ -0,0 +1,7 @@ +# Optional Linux tools for Windows hosts; contains no application secrets. +FROM docker:29.3.0-cli@sha256:067c301efe497cd5d174d468b7b3422a485ae4aca8f7ec1ffd4655c9fa383af2 AS cli +FROM golang:1.26.0-bookworm@sha256:2a0ba12e116687098780d3ce700f9ce3cb340783779646aafbabed748fa6677c +COPY --from=cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=cli /usr/local/libexec/docker/cli-plugins/docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose +ENV PYTHONUTF8=1 PYTHONUNBUFFERED=1 GOTELEMETRY=off +WORKDIR /workspace diff --git a/Makefile b/Makefile index 9ecb223..857ab25 100644 --- a/Makefile +++ b/Makefile @@ -24,3 +24,21 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-own test-infra explain test-race +test-own: + $(PYTHON) student-tests/acceptance.py --base-url "$(BASE_URL)" + $(PYTHON) student-tests/authentication.py --base-url "$(BASE_URL)" + +test-infra: + $(PYTHON) student-tests/infrastructure.py --base-url "$(BASE_URL)" + +explain: + $(PYTHON) student-tests/explain.py + +test-race: + CGO_ENABLED=1 go test -race ./... + +.PHONY: research +research: + $(PYTHON) student-tests/research.py diff --git a/REPORT.md b/REPORT.md index e42f2b0..7117a39 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,206 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Максим Ситосенко, БВТ2303. +- Целевая оценка: **5**, трек 2 — план запроса и восстановление. +- Commit/PR: ветка; коммит и PR созданы. Основа skeleton: `65003b6`. +- Источники: [PostgreSQL](https://www.postgresql.org/docs/17/using-explain.html), + [Valkey AOF](https://valkey.io/topics/persistence/), + [Argon2id](https://pkg.go.dev/golang.org/x/crypto/argon2), + [pg_dump](https://www.postgresql.org/docs/17/app-pgdump.html) / + [pg_restore](https://www.postgresql.org/docs/17/app-pgrestore.html). + Библиотеки: pgx/v5 5.7.6, go-redis/v9 9.12.1, x/crypto 0.43.0; полный lock — go.sum. + Codex использован для переноса реализации, тестов, проверки стенда и отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, Docker с Linux-контейнерами и Compose v2+, Python 3.9+, GNU make. +Сборка: Go 1.26.0; PostgreSQL 17.6; Valkey 8.1.3. Образы закреплены digest, +runtime API — scratch, UID 65532. Локальный Go нужен только для Go-тестов. +Команды ниже рассчитаны на fork после публикации файлов решения. + +```sh +git clone https://gitea.privcloud.tech/Roflyan/mtusi2026lab2-fork.git +cd mtusi2026lab2-fork +python3 -X utf8 scripts/bootstrap.py +docker compose config --quiet +docker compose up -d --build --wait +curl -f http://127.0.0.1:8080/health/ready +make check +make test BASE_URL=http://127.0.0.1:8080 +chmod +x scripts/restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 +make test-own BASE_URL=http://127.0.0.1:8080 +make test-infra BASE_URL=http://127.0.0.1:8080 +make explain +make research +``` + +Bootstrap создаёт `.env` и случайные секреты в `secrets/`, существующие значения +сохраняет. Миграции применяются API при старте; повтор вручную: +`docker compose run --rm --no-deps api migrate`. Не удалять секреты при сохранённых +volumes: пароль БД и ключ курсора должны оставаться прежними. + +Windows: использовать `python -X utf8` вместо `python3`; перед make задать +`$env:PYTHONUTF8="1"`. Без локального make/GCC: + +```powershell +docker build -f Dockerfile.tests -t mtusi-messenger:lab2-tests . +python -X utf8 scripts/linux_tests.py check +python -X utf8 scripts/linux_tests.py test +python -X utf8 scripts/linux_tests.py test-persistence +python -X utf8 scripts/linux_tests.py test-own +python -X utf8 scripts/linux_tests.py test-race +python -X utf8 student-tests/infrastructure.py +python -X utf8 student-tests/explain.py +python -X utf8 student-tests/research.py +``` + +Linux-runner выполняет исходные make targets; для persistence получает Docker +socket и запускает тот же `scripts/restart.sh`. Инфраструктурные тесты выполнять +последовательно, без параллельных пользователей стенда. Порты 8080 и 18082 свободны. Research запускается отдельно, ему нужен только Docker: +два временных PostgreSQL без сети, каждый с новым volume, 2 CPU и 512 MiB. +После проверки они удаляются, dump остаётся в игнорируемой папке `backups/`. + +Безопасная остановка: `docker compose down` — volumes остаются. +Повторный запуск: `docker compose up -d --build --wait`. +**Намеренное удаление всех учебных данных и сессий:** `docker compose down -v`. +Локальные секреты эта команда не удаляет. Команды ручной диагностики, полной +валидации OpenAPI и сохранения вывода — в [памятке проверок](student-tests/README.md). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → Go API → PostgreSQL (пользователи, чаты, участники, сообщения) и Valkey +(единственный источник сессий). Из lab1 перенесены API, Unicode-валидация, +проверка членства, подписанный cursor и graceful shutdown. Память процесса +заменена PostgreSQL; `X-User-Id` игнорируется, бизнес-запросы требуют Bearer. +Схема, выбранная до реализации запросов, сведена сюда: +`users` — PK id, UNIQUE username; `chats` — PK id; +`chat_members` — PK (chat_id, user_id) и FK на чат/пользователя; +`messages` — PK id, FK на чат и составной FK (chat_id, sender_id) на членство. +Индексы списка чатов: `chat_members(user_id, chat_id)`, `chats(created_at, id)`. +Миграции встроены в бинарник: версия и checksum в `schema_migrations`, +advisory lock от параллельного запуска, каждый файл — отдельная транзакция. +Свободные строки хранятся в UTF-8 bytea: это сохраняет в том числе U+0000, +который PostgreSQL text не принимает; длину в code points проверяет API. + +Регистрация защищена UNIQUE username, создание чата и участников — общей +транзакцией и FK. Сообщение подтверждается после COMMIT; чтения идут в PostgreSQL +без кэша, Read Committed, synchronous_commit=on. Индекс `(chat_id, created_at, id)` обслуживает keyset-пагинацию с +`LIMIT limit+1`. SQL параметризован, пул БД ограничен 8 соединениями; timeout +запроса 5с, SQL 3с. При SIGTERM API прекращает приём и завершает запросы до 8с. + +Пароли: Argon2id, случайная соль, 64 MiB / 3 прохода / 1 поток. Basic используется +только для входа. Bearer — случайные 256 бит; Valkey хранит SHA256 токена, +владельца и абсолютный expiry. TTL не продлевается чтением или restart; logout +удаляет ключ. AOF `appendfsync always`, volume и `noeviction` сохраняют сессии. +Отказ любой зависимости даёт 503; без сессии — 401. Readiness зависит от обоих +хранилищ, liveness — только от API. Логи не содержат тела, пароли или токены. + +`.env.example`: APP_HOST — bind API; APP_PORT — опубликованный порт (в контейнере +8080); APP_ENV — метка логов; SESSION_STORE допускает `valkey`; +SESSION_TTL_SECONDS — TTL; DB_POOL_MAX / MAX_INFLIGHT — лимиты; +`*_FILE` — пути смонтированных secrets. Максимум два одновременных KDF, +тело запроса ограничено 128 KiB. Дополнительные ответы: 413 при большом теле, +503 при перегрузке или недоступности зависимости, с общим форматом ошибки. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Неизменность skeleton | `git diff --exit-code 65003b6 -- contracts tests` (полная команда в памятке) | 17 обязательных файлов совпадают; публичные make targets сохранены | [source-check.txt](evidence/source-check.txt), [хеши](evidence/source-manifest.json) | +| Offline и публичный API | `make check`, `make test` | PASS | [check.txt](evidence/check.txt), [smoke.txt](evidence/smoke.txt) | +| Перенос lab1 | `make test-own` | JSON, Unicode, членство, пагинация; 100/100 уникальных сообщений | [own.txt](evidence/own.txt) | +| PostgreSQL, миграции с нуля | clone → bootstrap → `docker compose up -d --build --wait` → `make test` | Первичная сборка отдельного локального clone на новых volumes прошла | [clean-build.txt](evidence/clean-build.txt) | +| Данные и сессии переживают restart | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | Старый токен читает прежнее сообщение | [persistence](evidence/linux-test-persistence.txt), [до](evidence/restart-before.txt), [после](evidence/restart-after.txt) | +| Миграции повторяемы, чат атомарен | `make test-infra` | Два повтора без изменения данных; FK-ошибка не оставляет чат/участников | [infrastructure.json](evidence/infrastructure.json) | +| KDF, Basic/Bearer, logout | `make test-own` | Неверный пароль и X-User-Id не дают доступ; logout → 401 | [own.txt](evidence/own.txt) | +| Valkey TTL и persistence | `make test-infra`, persistence | TTL=2с → 401; отозванная сессия не оживает после restart | [infrastructure.json](evidence/infrastructure.json) | +| SQL, пул, гонка регистрации | `make test-own`; ревью store.go/main.go | Параметры отдельно от SQL; пул 8; 1×201 и 9×409 | [own.txt](evidence/own.txt) | +| Индекс на ≥10 000 сообщениях | `make explain` | Index Scan, 51 строка; нет Seq Scan/Sort | [explain.txt](evidence/explain.txt) | +| Отказ PostgreSQL/Valkey | `make test-infra` | Ограниченные 503; anonymous 401; восстановление без потери | [infrastructure.json](evidence/infrastructure.json) | +| Конкурентность и завершение | `make test-race`, `make test-infra` | Race detector пройден; SIGTERM завершился с кодом 0 | [race.txt](evidence/race.txt), [infrastructure.json](evidence/infrastructure.json) | +| Контейнер и секреты | `docker inspect`, `git check-ignore` (команды в памятке) | UID 65532, readonly, healthy; фактический AOF always; секреты исключены | [audit.json](evidence/audit.json) | +| Трек 5: индексы и стоимость записи | `make research` | 100 000 сообщений, 3 размера чатов, два индекса, 3 повтора | [исходные замеры](evidence/research.csv), [планы](evidence/research-plans.txt) | +| Трек 5: backup/restore | `make research` | Новый volume: совпали счётчики, 11 сообщений и SHA-256 всей истории | [research.json](evidence/research.json) | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +`clean-build.txt`, `audit.json` и `source-manifest.json` — результаты первичной +проверки. Служебные генераторы этих доказательств удалены при упрощении проекта; +повторять проверки можно обычными командами из отчёта и памятки. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза трека 2:** индекс `(chat_id, created_at, id)` сокращает чтение страницы +большого чата по сравнению с `(chat_id)`, но занимает больше места и требует +дополнительной работы при записи. Методика: `python -X utf8 student-tests/research.py`. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Стенд: PostgreSQL 17.6, Docker Desktop 4.66.0 / Engine 29.3.0, Linux/amd64, +12 доступных CPU и 3.68 GiB RAM. Данные синтетические: 100 000 сообщений в чатах +80 000 / 15 000 / 5 000, включая Unicode и байт U+0000. Источнику и восстановленной +БД выделены отдельные новые volumes, по 2 CPU / 512 MiB, сеть отключена. +Основной Compose-стенд не участвует в эксперименте. + +Сравниваются два индекса при неизменных PK/FK и запросе API: фильтр по чату и +`(created_at,id)`, сортировка по этой паре, LIMIT 51. Три повтора с порядком +AB / BA / AB; перед каждой вставкой таблица очищается, индекс создаётся заново. +После INSERT 100 000 строк выполняется ANALYZE. Чтение: 3 прогрева и 20 замеров +на случай в каждом повторе; один клиент, тёплый кэш, без parallel query и JIT. +EXPLAIN фиксирует исполнение, buffers и WAL; выбор индекса не форсируется. + +| Метрика | `(chat_id)` | `(chat_id, created_at, id)` | +| --- | --- | --- | +| Большой чат, первая страница: p50 / p95 / p99, мс | 25.392 / 27.725 / 31.140 | 0.036 / 0.069 / 0.087 | +| Большой чат, после 70 000: p50 / p95 / p99, мс | 8.729 / 9.807 / 14.786 | 0.035 / 0.072 / 0.084 | +| Малый чат, после 4 000: p50 / p95 / p99, мс | 0.651 / 0.755 / 0.945 | 0.379 / 0.665 / 0.971 | +| INSERT 100 000: медиана исполнения, мс | 1071.7 | 1166.0 | +| Размер исследуемого индекса, MiB | 0.63 | 6.54 | +| WAL при вставке: медиана, MiB | 39.96 | 43.61 | +| Shared buffers глубокой страницы: медиана | 2856 | 6 | + +В большом чате простой индекс приводит к Seq Scan + Sort, составной — к Index +Scan с чтением 51 строки. Для малого чата планировщик выбирает Bitmap Scan + Sort: +выигрыш меньше, p99 в этом прогоне немного выше. Составной индекс занимает +в 10.3 раза больше места; медиана записи выше на +8.8%, WAL — на 9.1%. +Разброс трёх вставок перекрывается: устойчивый размер надбавки не доказан. +Для API сохранён составной индекс, уже реализованный в миграции 001. + +**Восстановление:** pg_dump -Fc создаёт согласованный backup, pg_restore восстанавливает +его во второй контейнер с новым пустым volume (до restore — 0 public-таблиц). +Совпали: users=1, chats=3, members=3, messages=100 000, migrations=2, +размеры чатов, 11 выбранных сообщений и SHA-256 потока всей истории в порядке id. +Dump: 2.89 MiB, создание 0.58с; +pg_restore — 1.16с; от старта новой БД до завершения +проверки — 6.65с. Старт API в это время не входит. + +После dump в источнике отдельно зафиксирована ещё одна запись: там 100 001 строка, +после восстановления — 100 000, новой записи нет. Backup сохраняет состояние +снимка; последующие COMMIT без WAL-архива/PITR не восстановятся. Фиксированный +RPO не заявлен, расписание backup не реализовано. Сессии Valkey в этот dump не входят. + +Raw: [research.csv](evidence/research.csv); сводка/сверка: +[research.json](evidence/research.json); планы: [research-plans.txt](evidence/research-plans.txt). +Это SQL-эксперимент: Execution Time не включает planning, Docker и COMMIT; +полное время вставки с COMMIT/Docker отдельно записано в CSV. HTTP RPS и CPU/RSS +не измерялись. Для чтений по 60 замеров; p99 близок к максимуму и не является SLA. +Планы получены с конкретными параметрами; generic plans подготовленных запросов +pgx могут отличаться. Данные умещаются в RAM, холодный диск не исследовался. + +Проверки уровня 4 сохранены отдельно в `student-tests/infrastructure.py`: + +| Воздействие | Ожидание | Наблюдение | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Restart API+БД+Valkey | Старый токен и сообщение сохранены | Публичный persistence PASS | В пределах 90с теста / потерь проверяемых данных нет | +| Остановка PostgreSQL | Ограниченный 503, anonymous 401 | 503 за 5.00–5.03с | Готовность после start ≈1.53с / сообщение сохранено | +| Остановка Valkey | 503 без обхода авторизации | 503 за 1.02–1.03с | Готовность после start ≈0.25с / токен сохранён | +| TTL=2с; logout + restart | Недействительный токен → 401 | Оба сценария прошли | Переписка сохранена | +| Ошибка участника | Полный rollback | Счётчики до/после равны | Частичных данных нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Одна БД и один Valkey, без HA. Backup/restore проверен локально на синтетической +БД; автоматического расписания, внешнего хранения backup и PITR нет. Потеря +volumes уничтожает данные, не попавшие в backup; восстановление Valkey отдельно +не исследовалось. HTTP предназначен для локального стенда; удалённому нужен TLS. +Cursor не даёт snapshot между страницами: поздняя вставка или коррекция часов +может поместить запись перед пройденной границей. Эксперимент с тёплым кэшем и +одним клиентом не доказывает устойчивость под длительной конкурентной нагрузкой. diff --git a/auth.go b/auth.go new file mode 100644 index 0000000..663bc94 --- /dev/null +++ b/auth.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/redis/go-redis/v9" + "golang.org/x/crypto/argon2" +) + +func password(body map[string]json.RawMessage) (string, error) { + var p string + if json.Unmarshal(body["password"], &p) != nil || utf8.RuneCountInString(p) < 12 || utf8.RuneCountInString(p) > 128 { + return "", invalid() + } + return p, nil +} +func hashPassword(p string) string { + salt := make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + panic(err) + } + key := argon2.IDKey([]byte(p), salt, 3, 64*1024, 1, 32) + return "$argon2id$v=19$m=65536,t=3,p=1$" + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(key) +} +func verifyPassword(hash, p string) bool { + parts := strings.Split(hash, "$") + if len(parts) != 6 || parts[1] != "argon2id" || parts[2] != "v=19" || parts[3] != "m=65536,t=3,p=1" { + return false + } + salt, e := base64.RawStdEncoding.DecodeString(parts[4]) + if e != nil || len(salt) != 16 { + return false + } + want, e := base64.RawStdEncoding.DecodeString(parts[5]) + if e != nil || len(want) != 32 { + return false + } + actual := argon2.IDKey([]byte(p), salt, 3, 64*1024, 1, 32) + return subtle.ConstantTimeCompare(want, actual) == 1 +} +func (a *App) kdfSlot(ctx context.Context) error { + select { + case a.kdf <- struct{}{}: + return nil + case <-ctx.Done(): + return unavailable() + } +} + +type storedSession struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +func sessionKey(token string) string { + h := sha256.Sum256([]byte(token)) + return "sess:" + hex.EncodeToString(h[:]) +} +func (a *App) authenticate(ctx context.Context, r *http.Request) (string, string, error) { + values := r.Header.Values("Authorization") + if len(values) != 1 { + return "", "", unauthorized() + } + parts := strings.Fields(values[0]) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { + return "", "", unauthorized() + } + token, e := base64.RawURLEncoding.DecodeString(parts[1]) + if e != nil || len(token) != 32 { + return "", "", unauthorized() + } + key := sessionKey(parts[1]) + raw, err := a.sessions.Get(ctx, key).Bytes() + if errors.Is(err, redis.Nil) { + return "", "", unauthorized() + } + if err != nil { + return "", "", unavailable() + } + var session storedSession + if json.Unmarshal(raw, &session) != nil { + return "", "", unavailable() + } + if !time.Now().Before(session.ExpiresAt) { + return "", "", unauthorized() + } + if _, err = a.user(ctx, session.UserID); err != nil { + var ae apiError + if errors.As(err, &ae) && ae.status == 404 { + return "", "", unauthorized() + } + return "", "", err + } + return session.UserID, key, nil +} +func (a *App) login(ctx context.Context, r *http.Request) (int, any, error) { + if r.ContentLength > 0 || len(r.TransferEncoding) > 0 { + return 0, nil, invalid() + } + username, p, ok := r.BasicAuth() + if len(r.Header.Values("Authorization")) != 1 || !ok || !usernamePattern.MatchString(username) || !utf8.ValidString(p) || utf8.RuneCountInString(p) < 12 || utf8.RuneCountInString(p) > 128 { + return 0, nil, unauthorized() + } + var hash string + var u User + var created time.Time + err := a.db.QueryRow(ctx, "SELECT id::text,username,display_name,created_at,password_hash FROM users WHERE username=$1", username).Scan(&u.ID, &u.Username, utf8String{&u.DisplayName}, &created, &hash) + absent := errors.Is(err, pgx.ErrNoRows) + if err != nil && !absent { + return 0, nil, unavailable() + } + if absent { + hash = a.dummyHash + } + if err = a.kdfSlot(ctx); err != nil { + return 0, nil, err + } + valid := verifyPassword(hash, p) + <-a.kdf + if !valid || absent { + return 0, nil, unauthorized() + } + u.CreatedAt = stamp(created) + tokenBytes := make([]byte, 32) + if _, err = rand.Read(tokenBytes); err != nil { + return 0, nil, unavailable() + } + token := base64.RawURLEncoding.EncodeToString(tokenBytes) + expires := time.Now().UTC().Truncate(time.Second).Add(a.ttl) + raw, _ := json.Marshal(storedSession{u.ID, expires}) + if err = a.sessions.SetArgs(ctx, sessionKey(token), raw, redis.SetArgs{Mode: "NX", ExpireAt: expires}).Err(); err != nil { + return 0, nil, unavailable() + } + return 201, map[string]any{"token": token, "token_type": "Bearer", "expires_at": expires.Format(time.RFC3339), "user": u}, nil +} diff --git a/auth_test.go b/auth_test.go new file mode 100644 index 0000000..0ee9519 --- /dev/null +++ b/auth_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "net/http/httptest" + "strings" + "testing" +) + +func TestPasswordHashSaltAndFullUnicode(t *testing.T) { + p := strings.Repeat("😀", 127) + ":" + a, b := hashPassword(p), hashPassword(p) + if a == b { + t.Fatal("salt reused") + } + if !verifyPassword(a, p) || verifyPassword(a, strings.Repeat("😀", 127)+"!") { + t.Fatal("password truncated or incorrect") + } + for _, bad := range []string{"", p, strings.Replace(a, "m=65536", "m=1", 1), a[:len(a)-5]} { + if verifyPassword(bad, p) { + t.Fatal("invalid hash accepted") + } + } +} + +func TestCursorIntegrityResourceAndRestart(t *testing.T) { + a := &App{secret: []byte("stable-local-test-secret-32-bytes")} + encoded := a.encodeCursor("messages:one", "2026-01-01T00:00:00.000000Z", uuid()) + restarted := &App{secret: append([]byte(nil), a.secret...)} + req := httptest.NewRequest("GET", "/messages?cursor="+encoded+"&limit=7", nil) + if limit, _, e := restarted.pagination(req, "messages:one"); e != nil || limit != 7 { + t.Fatal("cursor lost across restart") + } + if _, _, e := a.pagination(req, "messages:two"); e == nil { + t.Fatal("cross-resource cursor accepted") + } + forged := httptest.NewRequest("GET", "/messages?cursor="+encoded[:len(encoded)-4]+"AAAA", nil) + if _, _, e := a.pagination(forged, "messages:one"); e == nil { + t.Fatal("forged cursor accepted") + } +} + +func TestStrictJSON(t *testing.T) { + for _, raw := range []string{`{"text":"a","text":"b"}`, `{"text":"\ud800"}`, `{} {}`, `{"unknown":1}`} { + req := httptest.NewRequest("POST", "/", strings.NewReader(raw)) + req.Header.Set("Content-Type", "application/json") + if _, e := readObject(httptest.NewRecorder(), req, 1024, []string{"text"}); e == nil { + t.Fatalf("accepted %q", raw) + } + } +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..8edc9ef --- /dev/null +++ b/compose.yaml @@ -0,0 +1,62 @@ +name: mtusi-lab2 +services: + db: + image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + valkey: + image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 + command: [valkey-server, /run/secrets/valkey_config] + secrets: [valkey_config, valkey_password] + volumes: [sessions:/data] + healthcheck: + test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/valkey_password)" valkey-cli ping | grep -q PONG'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 256m + api: + image: mtusi-messenger:lab2 + build: . + ports: ["127.0.0.1:${APP_PORT:-8080}:8080"] + environment: + APP_HOST: ${APP_HOST:-0.0.0.0} + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_STORE: ${SESSION_STORE:-valkey} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + MAX_INFLIGHT: ${MAX_INFLIGHT:-32} + DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url} + CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url} + CURSOR_SECRET_FILE: ${CURSOR_SECRET_FILE:-/run/secrets/cursor_secret} + secrets: [database_url, cache_url, cursor_secret] + depends_on: + db: {condition: service_healthy} + valkey: {condition: service_healthy} + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + mem_limit: 512m + pids_limit: 128 + stop_grace_period: 10s +volumes: + pgdata: + sessions: +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + valkey_password: {file: ./secrets/valkey_password} + valkey_config: {file: ./secrets/valkey.conf} + cache_url: {file: ./secrets/cache_url} + cursor_secret: {file: ./secrets/cursor_secret} diff --git a/evidence/audit.json b/evidence/audit.json new file mode 100644 index 0000000..8720d9a --- /dev/null +++ b/evidence/audit.json @@ -0,0 +1,19 @@ +{ + "result": "PASS", + "git_ignores_private_files": true, + "public_files_scanned": 77, + "local_secrets_and_raw_bearer_absent": true, + "all_passwords_argon2id": true, + "api_user": "65532:65532", + "api_readonly": true, + "api_pid1": "/messenger", + "health": "healthy", + "valkey_effective_configuration": { + "appendfsync": "always", + "appendonly": "yes", + "maxmemory-policy": "noeviction" + }, + "engine": "29.3.0", + "compose": "5.1.0", + "host": "linux/x86_64 CPUs=12 MemoryBytes=3956064256" +} diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..ddca30b --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,5 @@ +2026-10-16T11:21:37Z +$ make check +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +exit_code=0 diff --git a/evidence/clean-build.txt b/evidence/clean-build.txt new file mode 100644 index 0000000..94fa73c --- /dev/null +++ b/evidence/clean-build.txt @@ -0,0 +1,147 @@ +2026-10-16T11:16:54Z +$ python -X utf8 scripts/clean_build.py +warning: in the working copy of '.dockerignore', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of '.env.example', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of '.gitattributes', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of '.github/pull_request_template.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'ARCHITECTURE.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'Dockerfile', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'Dockerfile.tests', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'REPORT.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'compose.yaml', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/acceptance.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/audit.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/authentication.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/check.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/clean-build.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/explain-run.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/infrastructure.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/linux-test-persistence.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/own.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/persistence.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/race.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/restart-after.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/restart-before.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/smoke.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'evidence/source-check.txt', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'go.mod', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'go.sum', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'scripts/audit.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'scripts/bootstrap.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'scripts/clean_build.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'scripts/linux_tests.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'scripts/record.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'scripts/restart.cmd', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'scripts/restart.example.sh', CRLF will be replaced by LF the next time Git touches it +warning: in the working copy of 'scripts/verify_sources.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'student-tests/README.md', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'student-tests/acceptance.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'student-tests/authentication.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'student-tests/explain.py', LF will be replaced by CRLF the next time Git touches it +warning: in the working copy of 'student-tests/infrastructure.py', LF will be replaced by CRLF the next time Git touches it +CLEAN CLONE: temporary snapshot commit d8ebfe039d149590cddfb4337b1698b1768686fe +MAIN REPOSITORY: no commit, no staging, no push +Local .env and secrets ready; existing values preserved. + Image mtusi-messenger:lab2 Building +#1 [internal] load local bake definitions +#1 reading from stdin 612B 0.1s done +#1 DONE 0.1s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 624B 0.0s done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/golang:1.26.0-bookworm@sha256:2a0ba12e116687098780d3ce700f9ce3cb340783779646aafbabed748fa6677c +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 115B done +#4 DONE 0.0s + +#5 [build 1/7] FROM docker.io/library/golang:1.26.0-bookworm@sha256:2a0ba12e116687098780d3ce700f9ce3cb340783779646aafbabed748fa6677c +#5 DONE 0.0s + +#6 [internal] load build context +#6 transferring context: 35.68kB 0.1s done +#6 DONE 0.1s + +#7 [build 2/7] WORKDIR /src +#7 CACHED + +#8 [build 3/7] COPY go.mod go.sum ./ +#8 CACHED + +#9 [build 4/7] RUN go mod download +#9 CACHED + +#10 [build 5/7] COPY *.go ./ +#10 DONE 0.1s + +#11 [build 6/7] COPY migrations ./migrations +#11 DONE 0.1s + +#12 [build 7/7] RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /messenger . +#12 DONE 35.8s + +#13 [runtime 1/1] COPY --from=build /messenger /messenger +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 writing image sha256:81a8f7752dad98518b6154841b4d6382fbdf9b6f849cb6aed11b65b58bd0125e 0.0s done +#14 naming to docker.io/library/mtusi-messenger:lab2 0.0s done +#14 DONE 0.1s + +#15 resolving provenance for metadata file +#15 DONE 0.1s + Image mtusi-messenger:lab2 Built + Network lab2-clean-aa70a5f3_default Creating + Network lab2-clean-aa70a5f3_default Created + Volume lab2-clean-aa70a5f3_pgdata Creating + Volume lab2-clean-aa70a5f3_pgdata Created + Volume lab2-clean-aa70a5f3_sessions Creating + Volume lab2-clean-aa70a5f3_sessions Created + Container lab2-clean-aa70a5f3-db-1 Creating + Container lab2-clean-aa70a5f3-valkey-1 Creating + Container lab2-clean-aa70a5f3-valkey-1 Created + Container lab2-clean-aa70a5f3-db-1 Created + Container lab2-clean-aa70a5f3-api-1 Creating + Container lab2-clean-aa70a5f3-api-1 Created + Container lab2-clean-aa70a5f3-valkey-1 Starting + Container lab2-clean-aa70a5f3-db-1 Starting + Container lab2-clean-aa70a5f3-db-1 Started + Container lab2-clean-aa70a5f3-valkey-1 Started + Container lab2-clean-aa70a5f3-db-1 Waiting + Container lab2-clean-aa70a5f3-valkey-1 Waiting + Container lab2-clean-aa70a5f3-valkey-1 Healthy + Container lab2-clean-aa70a5f3-db-1 Healthy + Container lab2-clean-aa70a5f3-api-1 Starting + Container lab2-clean-aa70a5f3-api-1 Started + Container lab2-clean-aa70a5f3-db-1 Waiting + Container lab2-clean-aa70a5f3-valkey-1 Waiting + Container lab2-clean-aa70a5f3-api-1 Waiting + Container lab2-clean-aa70a5f3-valkey-1 Healthy + Container lab2-clean-aa70a5f3-db-1 Healthy + Container lab2-clean-aa70a5f3-api-1 Healthy +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18083" +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +EMPTY VOLUME MIGRATIONS: 001_initial.sql +002_unicode_storage.sql + Container lab2-clean-aa70a5f3-api-1 Stopping + Container lab2-clean-aa70a5f3-api-1 Stopped + Container lab2-clean-aa70a5f3-api-1 Removing + Container lab2-clean-aa70a5f3-api-1 Removed + Container lab2-clean-aa70a5f3-db-1 Stopping + Container lab2-clean-aa70a5f3-valkey-1 Stopping + Container lab2-clean-aa70a5f3-valkey-1 Stopped + Container lab2-clean-aa70a5f3-valkey-1 Removing + Container lab2-clean-aa70a5f3-valkey-1 Removed + Container lab2-clean-aa70a5f3-db-1 Stopped + Container lab2-clean-aa70a5f3-db-1 Removing + Container lab2-clean-aa70a5f3-db-1 Removed + Network lab2-clean-aa70a5f3_default Removing + Network lab2-clean-aa70a5f3_default Removed +PASS: isolated clone build; temporary project stopped, volumes retained +exit_code=0 diff --git a/evidence/explain-run.txt b/evidence/explain-run.txt new file mode 100644 index 0000000..bd7d9e3 --- /dev/null +++ b/evidence/explain-run.txt @@ -0,0 +1,6 @@ +2026-10-16T11:19:39Z +$ make explain +python3 student-tests/explain.py +PASS: 10000 messages; first and deep page use messages_history_idx without Seq Scan/Sort +evidence\explain.txt +exit_code=0 diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 0000000..a86235e --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,32 @@ +PostgreSQL: PostgreSQL 17.6 (Debian 17.6-2.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit +Rows in dedicated chat: 10000 + +first +EXPLAIN (ANALYZE, BUFFERS) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='aff35ede-28d0-4df7-b577-2a29a956ba40' AND (created_at,id)>('2026-01-01T00:00:00Z'::timestamptz,'00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=0.41..16.09 rows=51 width=146) (actual time=0.033..0.045 rows=51 loops=1) + Buffers: shared hit=4 + -> Index Scan using messages_history_idx on messages (cost=0.41..3074.29 rows=9998 width=146) (actual time=0.031..0.041 rows=51 loops=1) + Index Cond: ((chat_id = 'aff35ede-28d0-4df7-b577-2a29a956ba40'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Buffers: shared hit=4 +Planning: + Buffers: shared hit=116 +Planning Time: 0.981 ms +Execution Time: 0.381 ms + +deep (after 9000 rows) +EXPLAIN (ANALYZE, BUFFERS) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='aff35ede-28d0-4df7-b577-2a29a956ba40' AND (created_at,id)>('2026-01-01T00:00:09Z'::timestamptz,'c4fd0d86-6dcf-4177-a4df-90cebe00e335'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=0.41..108.20 rows=51 width=146) (actual time=0.026..0.040 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history_idx on messages (cost=0.41..2367.56 rows=1120 width=146) (actual time=0.025..0.037 rows=51 loops=1) + Index Cond: ((chat_id = 'aff35ede-28d0-4df7-b577-2a29a956ba40'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:09+00'::timestamp with time zone, 'c4fd0d86-6dcf-4177-a4df-90cebe00e335'::uuid))) + Buffers: shared hit=5 +Planning: + Buffers: shared hit=112 +Planning Time: 0.221 ms +Execution Time: 0.071 ms diff --git a/evidence/infrastructure.json b/evidence/infrastructure.json new file mode 100644 index 0000000..9f476e6 --- /dev/null +++ b/evidence/infrastructure.json @@ -0,0 +1,55 @@ +{ + "utc": "2026-10-16T11:16:50Z", + "outages": [ + { + "service": "db", + "statuses": [ + 503, + 503, + 401, + 200, + 503 + ], + "request_seconds": [ + 5.0, + 5.031 + ], + "ready_seconds_after_start": 1.531, + "started_before": "2026-10-16T11:11:04.861601683Z", + "started_after": "2026-10-16T11:17:13.132832575Z", + "same_token_and_message": true + }, + { + "service": "valkey", + "statuses": [ + 503, + 503, + 401, + 200, + 503 + ], + "request_seconds": [ + 1.031, + 1.016 + ], + "ready_seconds_after_start": 0.25, + "started_before": "2026-10-16T11:11:04.901502042Z", + "started_after": "2026-10-16T11:17:23.47318257Z", + "same_token_and_message": true + } + ], + "rollback": { + "before": "148|78|140|30553|2", + "after": "148|78|140|30553|2", + "result": "PASS: no chat or membership left" + }, + "migrations": "PASS: two repeated migrations, counts unchanged", + "expiry": "PASS: separate API TTL=2s, 200 then 401 after 2.2s", + "sigterm": { + "seconds": 1.219, + "exit_code": 0, + "oom_killed": false, + "no_new_requests": true + }, + "logout_restart": "PASS: revoked session stays revoked after Valkey restart" +} diff --git a/evidence/infrastructure.txt b/evidence/infrastructure.txt new file mode 100644 index 0000000..e9e1b69 --- /dev/null +++ b/evidence/infrastructure.txt @@ -0,0 +1,10 @@ +2026-10-16T11:16:40Z +$ make test-infra BASE_URL=http://127.0.0.1:8080 +python3 student-tests/infrastructure.py --base-url "http://127.0.0.1:8080" +ROLLBACK: table counts unchanged after FK failure +MIGRATIONS: repeat twice, counts unchanged +DB: bounded 503, anonymous 401, recovery with same token/message +VALKEY: bounded 503, anonymous 401, recovery with same token/message +EXPIRY: TTL=2s, 200 then 401 +PASS: infrastructure, expiry, rollback, migrations, shutdown and durable logout +exit_code=0 diff --git a/evidence/linux-test-persistence.txt b/evidence/linux-test-persistence.txt new file mode 100644 index 0000000..1632654 --- /dev/null +++ b/evidence/linux-test-persistence.txt @@ -0,0 +1,6 @@ +2026-10-16T11:18:44Z +$ make test-persistence BASE_URL=http://api:8080 ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://api:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +exit_code=0 diff --git a/evidence/own.txt b/evidence/own.txt new file mode 100644 index 0000000..a211475 --- /dev/null +++ b/evidence/own.txt @@ -0,0 +1,29 @@ +2026-10-16T11:15:12Z +$ make test-own BASE_URL=http://127.0.0.1:8080 +python3 student-tests/acceptance.py --base-url "http://127.0.0.1:8080" +test_100_messages_from_10_clients (__main__.Acceptance.test_100_messages_from_10_clients) ... CONCURRENCY: 10 clients x 10 writes = 100 acknowledged, 100 read, 100 unique +ok +test_chat_cursor_bound_to_current_user (__main__.Acceptance.test_chat_cursor_bound_to_current_user) ... ok +test_duplicate_username_and_user_validation (__main__.Acceptance.test_duplicate_username_and_user_validation) ... ok +test_empty_and_unicode_message (__main__.Acceptance.test_empty_and_unicode_message) ... ok +test_empty_history_default_limit_and_concurrent_append (__main__.Acceptance.test_empty_history_default_limit_and_concurrent_append) ... ok +test_foreign_chat_and_authentication (__main__.Acceptance.test_foreign_chat_and_authentication) ... ok +test_invalid_json (__main__.Acceptance.test_invalid_json) ... ok +test_membership_validation (__main__.Acceptance.test_membership_validation) ... ok +test_multiple_pages_and_cursor_scope (__main__.Acceptance.test_multiple_pages_and_cursor_scope) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 16.794s + +OK +python3 student-tests/authentication.py --base-url "http://127.0.0.1:8080" +test_password_boundaries_unicode_and_colon (__main__.Authentication.test_password_boundaries_unicode_and_colon) ... ok +test_registration_race (__main__.Authentication.test_registration_race) ... REGISTRATION RACE: 10 requests, 1 created, 9 conflicts +ok +test_wrong_password_logout_and_identity (__main__.Authentication.test_wrong_password_logout_and_identity) ... ok + +---------------------------------------------------------------------- +Ran 3 tests in 3.026s + +OK +exit_code=0 diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 0000000..30ee8bb --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,16 @@ +2026-10-16T11:17:43Z +$ make test-race +CGO_ENABLED=1 go test -race ./... +go: downloading github.com/redis/go-redis/v9 v9.12.1 +go: downloading github.com/jackc/pgx/v5 v5.7.6 +go: downloading golang.org/x/crypto v0.43.0 +go: downloading github.com/cespare/xxhash/v2 v2.3.0 +go: downloading github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f +go: downloading golang.org/x/text v0.30.0 +go: downloading github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 +go: downloading github.com/jackc/pgpassfile v1.0.0 +go: downloading github.com/jackc/puddle/v2 v2.2.2 +go: downloading golang.org/x/sys v0.37.0 +go: downloading golang.org/x/sync v0.17.0 +ok mtusi-messenger 6.980s +exit_code=0 diff --git a/evidence/research-plans.txt b/evidence/research-plans.txt new file mode 100644 index 0000000..f23d594 --- /dev/null +++ b/evidence/research-plans.txt @@ -0,0 +1,129 @@ +chat/insert_100000 (EXPLAIN JSON) +{"Plan": {"Node Type": "ModifyTable", "Operation": "Insert", "Parallel Aware": false, "Async Capable": false, "Relation Name": "messages", "Alias": "messages", "Startup Cost": 0.0, "Total Cost": 5750.0, "Plan Rows": 0, "Plan Width": 0, "Actual Startup Time": 991.526, "Actual Total Time": 991.527, "Actual Rows": 0, "Actual Loops": 1, "Shared Hit Blocks": 545161, "Shared Read Blocks": 2, "Shared Dirtied Blocks": 3466, "Shared Written Blocks": 3466, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 171, "Temp Written Blocks": 171, "WAL Records": 301651, "WAL FPI": 0, "WAL Bytes": 41896134, "Plans": [{"Node Type": "Function Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Function Name": "generate_series", "Alias": "g", "Startup Cost": 0.0, "Total Cost": 5750.0, "Plan Rows": 100000, "Plan Width": 88, "Actual Startup Time": 11.145, "Actual Total Time": 172.283, "Actual Rows": 100000, "Actual Loops": 1, "Shared Hit Blocks": 0, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 171, "Temp Written Blocks": 171, "WAL Records": 0, "WAL FPI": 0, "WAL Bytes": 0}]}, "Planning": {"Shared Hit Blocks": 15, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0}, "Planning Time": 0.141, "Triggers": [{"Trigger Name": "RI_ConstraintTrigger_c_16443", "Constraint Name": "messages_chat_id_fkey", "Relation": "messages", "Time": 301.956, "Calls": 100000}, {"Trigger Name": "RI_ConstraintTrigger_c_16448", "Constraint Name": "messages_chat_id_sender_id_fkey", "Relation": "messages", "Time": 319.258, "Calls": 100000}], "Execution Time": 1623.789} + +chat/hot_first +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='6adca606-cb42-ce80-9dfe-e9c5ab3f1b51' AND (created_at,id)>( + '2026-01-01T00:00:00Z'::timestamptz+0*interval '1 millisecond','00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=8700.40..8700.53 rows=51 width=257) (actual time=30.996..31.003 rows=51 loops=1) + Buffers: shared hit=2862 + -> Sort (cost=8700.40..8899.13 rows=79492 width=257) (actual time=30.994..30.998 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 50kB + Buffers: shared hit=2862 + -> Seq Scan on messages (cost=0.00..6048.38 rows=79492 width=257) (actual time=0.018..20.924 rows=80000 loops=1) + Filter: ((chat_id = '6adca606-cb42-ce80-9dfe-e9c5ab3f1b51'::uuid) AND (ROW(created_at, id) > ROW(('2026-01-01 00:00:00+00'::timestamp with time zone + '00:00:00'::interval), '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 20000 + Buffers: shared hit=2856 +Planning: + Buffers: shared hit=101 +Planning Time: 0.309 ms +Execution Time: 31.050 ms + +chat/hot_deep +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='6adca606-cb42-ce80-9dfe-e9c5ab3f1b51' AND (created_at,id)>( + '2026-01-01T00:00:00Z'::timestamptz+70000*interval '1 millisecond','51007766-120b-9354-54a6-dc1c61f4f185'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=6024.82..6024.94 rows=51 width=257) (actual time=14.114..14.122 rows=51 loops=1) + Buffers: shared hit=2862 + -> Sort (cost=6024.82..6085.24 rows=24168 width=257) (actual time=14.112..14.116 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 50kB + Buffers: shared hit=2862 + -> Seq Scan on messages (cost=0.00..5218.52 rows=24168 width=257) (actual time=8.013..12.721 rows=10000 loops=1) + Filter: ((chat_id = '6adca606-cb42-ce80-9dfe-e9c5ab3f1b51'::uuid) AND (ROW(created_at, id) > ROW(('2026-01-01 00:00:00+00'::timestamp with time zone + '00:01:10'::interval), '51007766-120b-9354-54a6-dc1c61f4f185'::uuid))) + Rows Removed by Filter: 90000 + Buffers: shared hit=2856 +Planning: + Buffers: shared hit=101 +Planning Time: 0.247 ms +Execution Time: 14.167 ms + +chat/small_deep +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='0edbe356-1e20-7355-3db8-96e0eb8b4f65' AND (created_at,id)>( + '2026-01-01T00:00:00Z'::timestamptz+99000*interval '1 millisecond','ed7d9f1e-ca01-40c0-5d78-abf027268fa3'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=3091.06..3091.19 rows=51 width=257) (actual time=0.721..0.725 rows=51 loops=1) + Buffers: shared hit=155 + -> Sort (cost=3091.06..3091.19 rows=54 width=257) (actual time=0.720..0.722 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 50kB + Buffers: shared hit=155 + -> Bitmap Heap Scan on messages (cost=58.38..3089.50 rows=54 width=257) (actual time=0.387..0.581 rows=1000 loops=1) + Recheck Cond: (chat_id = '0edbe356-1e20-7355-3db8-96e0eb8b4f65'::uuid) + Filter: (ROW(created_at, id) > ROW(('2026-01-01 00:00:00+00'::timestamp with time zone + '00:01:39'::interval), 'ed7d9f1e-ca01-40c0-5d78-abf027268fa3'::uuid)) + Rows Removed by Filter: 4000 + Heap Blocks: exact=144 + Buffers: shared hit=149 + -> Bitmap Index Scan on research_chat_idx (cost=0.00..58.37 rows=5077 width=0) (actual time=0.070..0.071 rows=5000 loops=1) + Index Cond: (chat_id = '0edbe356-1e20-7355-3db8-96e0eb8b4f65'::uuid) + Buffers: shared hit=5 +Planning: + Buffers: shared hit=101 +Planning Time: 0.210 ms +Execution Time: 0.762 ms + +composite/insert_100000 (EXPLAIN JSON) +{"Plan": {"Node Type": "ModifyTable", "Operation": "Insert", "Parallel Aware": false, "Async Capable": false, "Relation Name": "messages", "Alias": "messages", "Startup Cost": 0.0, "Total Cost": 5750.0, "Plan Rows": 0, "Plan Width": 0, "Actual Startup Time": 656.964, "Actual Total Time": 656.965, "Actual Rows": 0, "Actual Loops": 1, "Shared Hit Blocks": 520819, "Shared Read Blocks": 2, "Shared Dirtied Blocks": 4222, "Shared Written Blocks": 4222, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 171, "Temp Written Blocks": 171, "WAL Records": 301359, "WAL FPI": 0, "WAL Bytes": 45729460, "Plans": [{"Node Type": "Function Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Function Name": "generate_series", "Alias": "g", "Startup Cost": 0.0, "Total Cost": 5750.0, "Plan Rows": 100000, "Plan Width": 88, "Actual Startup Time": 7.165, "Actual Total Time": 145.253, "Actual Rows": 100000, "Actual Loops": 1, "Shared Hit Blocks": 0, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 171, "Temp Written Blocks": 171, "WAL Records": 0, "WAL FPI": 0, "WAL Bytes": 0}]}, "Planning": {"Shared Hit Blocks": 15, "Shared Read Blocks": 0, "Shared Dirtied Blocks": 0, "Shared Written Blocks": 0, "Local Hit Blocks": 0, "Local Read Blocks": 0, "Local Dirtied Blocks": 0, "Local Written Blocks": 0, "Temp Read Blocks": 0, "Temp Written Blocks": 0}, "Planning Time": 0.104, "Triggers": [{"Trigger Name": "RI_ConstraintTrigger_c_16443", "Constraint Name": "messages_chat_id_fkey", "Relation": "messages", "Time": 298.604, "Calls": 100000}, {"Trigger Name": "RI_ConstraintTrigger_c_16448", "Constraint Name": "messages_chat_id_sender_id_fkey", "Relation": "messages", "Time": 321.92, "Calls": 100000}], "Execution Time": 1289.336} + +composite/hot_first +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='6adca606-cb42-ce80-9dfe-e9c5ab3f1b51' AND (created_at,id)>( + '2026-01-01T00:00:00Z'::timestamptz+0*interval '1 millisecond','00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=0.42..11.17 rows=51 width=257) (actual time=0.032..0.047 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history_idx on messages (cost=0.42..16911.78 rows=80205 width=257) (actual time=0.032..0.044 rows=51 loops=1) + Index Cond: ((chat_id = '6adca606-cb42-ce80-9dfe-e9c5ab3f1b51'::uuid) AND (ROW(created_at, id) > ROW(('2026-01-01 00:00:00+00'::timestamp with time zone + '00:00:00'::interval), '00000000-0000-0000-0000-000000000000'::uuid))) + Buffers: shared hit=5 +Planning: + Buffers: shared hit=112 +Planning Time: 0.264 ms +Execution Time: 0.078 ms + +composite/hot_deep +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='6adca606-cb42-ce80-9dfe-e9c5ab3f1b51' AND (created_at,id)>( + '2026-01-01T00:00:00Z'::timestamptz+70000*interval '1 millisecond','51007766-120b-9354-54a6-dc1c61f4f185'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=0.42..28.28 rows=51 width=257) (actual time=0.038..0.051 rows=51 loops=1) + Buffers: shared hit=6 + -> Index Scan using messages_history_idx on messages (cost=0.42..13055.69 rows=23899 width=257) (actual time=0.037..0.047 rows=51 loops=1) + Index Cond: ((chat_id = '6adca606-cb42-ce80-9dfe-e9c5ab3f1b51'::uuid) AND (ROW(created_at, id) > ROW(('2026-01-01 00:00:00+00'::timestamp with time zone + '00:01:10'::interval), '51007766-120b-9354-54a6-dc1c61f4f185'::uuid))) + Buffers: shared hit=6 +Planning: + Buffers: shared hit=112 +Planning Time: 0.248 ms +Execution Time: 0.083 ms + +composite/small_deep +EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='0edbe356-1e20-7355-3db8-96e0eb8b4f65' AND (created_at,id)>( + '2026-01-01T00:00:00Z'::timestamptz+99000*interval '1 millisecond','ed7d9f1e-ca01-40c0-5d78-abf027268fa3'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51; +Limit (cost=170.75..170.86 rows=45 width=257) (actual time=0.463..0.468 rows=51 loops=1) + Buffers: shared hit=50 + -> Sort (cost=170.75..170.86 rows=45 width=257) (actual time=0.462..0.465 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 50kB + Buffers: shared hit=50 + -> Bitmap Heap Scan on messages (cost=4.88..169.51 rows=45 width=257) (actual time=0.115..0.305 rows=1000 loops=1) + Recheck Cond: ((chat_id = '0edbe356-1e20-7355-3db8-96e0eb8b4f65'::uuid) AND (ROW(created_at, id) > ROW(('2026-01-01 00:00:00+00'::timestamp with time zone + '00:01:39'::interval), 'ed7d9f1e-ca01-40c0-5d78-abf027268fa3'::uuid))) + Heap Blocks: exact=29 + Buffers: shared hit=44 + -> Bitmap Index Scan on messages_history_idx (cost=0.00..4.87 rows=45 width=0) (actual time=0.105..0.105 rows=1000 loops=1) + Index Cond: ((chat_id = '0edbe356-1e20-7355-3db8-96e0eb8b4f65'::uuid) AND (ROW(created_at, id) > ROW(('2026-01-01 00:00:00+00'::timestamp with time zone + '00:01:39'::interval), 'ed7d9f1e-ca01-40c0-5d78-abf027268fa3'::uuid))) + Buffers: shared hit=15 +Planning: + Buffers: shared hit=112 +Planning Time: 0.290 ms +Execution Time: 0.512 ms diff --git a/evidence/research.csv b/evidence/research.csv new file mode 100644 index 0000000..ec9d4f3 --- /dev/null +++ b/evidence/research.csv @@ -0,0 +1,367 @@ +repeat,variant,case,execution_ms,wall_with_commit_and_docker_ms,wal_bytes,index_bytes,planning_ms,shared_hit_blocks,shared_read_blocks +1,chat,insert_100000,1623.789,1860.107299988158,41896134,663552,,, +1,chat,hot_first,26.09,,,,0.242,2862,0 +1,chat,hot_first,24.815,,,,0.068,2856,0 +1,chat,hot_first,25.321,,,,0.1,2856,0 +1,chat,hot_first,27.364,,,,0.091,2856,0 +1,chat,hot_first,26.19,,,,0.38,2856,0 +1,chat,hot_first,24.915,,,,0.068,2856,0 +1,chat,hot_first,24.341,,,,0.065,2856,0 +1,chat,hot_first,25.239,,,,0.068,2856,0 +1,chat,hot_first,25.957,,,,0.12,2856,0 +1,chat,hot_first,25.037,,,,0.066,2856,0 +1,chat,hot_first,24.031,,,,0.065,2856,0 +1,chat,hot_first,24.805,,,,0.065,2856,0 +1,chat,hot_first,26.653,,,,0.233,2856,0 +1,chat,hot_first,25.373,,,,0.076,2856,0 +1,chat,hot_first,25.078,,,,0.069,2856,0 +1,chat,hot_first,25.93,,,,0.099,2856,0 +1,chat,hot_first,26.354,,,,0.1,2856,0 +1,chat,hot_first,25.725,,,,0.068,2856,0 +1,chat,hot_first,25.411,,,,0.066,2856,0 +1,chat,hot_first,26.59,,,,0.067,2856,0 +1,chat,hot_deep,11.36,,,,0.215,2862,0 +1,chat,hot_deep,8.917,,,,0.069,2856,0 +1,chat,hot_deep,8.863,,,,0.067,2856,0 +1,chat,hot_deep,9.008,,,,0.067,2856,0 +1,chat,hot_deep,8.587,,,,0.068,2856,0 +1,chat,hot_deep,9.766,,,,0.069,2856,0 +1,chat,hot_deep,8.607,,,,0.068,2856,0 +1,chat,hot_deep,9.304,,,,0.075,2856,0 +1,chat,hot_deep,9.045,,,,0.07,2856,0 +1,chat,hot_deep,8.578,,,,0.069,2856,0 +1,chat,hot_deep,10.621,,,,0.068,2856,0 +1,chat,hot_deep,8.722,,,,0.07,2856,0 +1,chat,hot_deep,9.06,,,,0.069,2856,0 +1,chat,hot_deep,8.834,,,,0.069,2856,0 +1,chat,hot_deep,9.253,,,,0.069,2856,0 +1,chat,hot_deep,8.762,,,,0.068,2856,0 +1,chat,hot_deep,8.679,,,,0.069,2856,0 +1,chat,hot_deep,9.126,,,,0.067,2856,0 +1,chat,hot_deep,9.439,,,,0.072,2856,0 +1,chat,hot_deep,9.336,,,,0.181,2856,0 +1,chat,small_deep,0.836,,,,0.214,155,0 +1,chat,small_deep,0.745,,,,0.042,149,0 +1,chat,small_deep,0.671,,,,0.048,149,0 +1,chat,small_deep,0.68,,,,0.039,149,0 +1,chat,small_deep,0.643,,,,0.042,149,0 +1,chat,small_deep,0.646,,,,0.041,149,0 +1,chat,small_deep,0.582,,,,0.05,149,0 +1,chat,small_deep,0.603,,,,0.04,149,0 +1,chat,small_deep,0.585,,,,0.042,149,0 +1,chat,small_deep,0.631,,,,0.042,149,0 +1,chat,small_deep,0.611,,,,0.048,149,0 +1,chat,small_deep,0.58,,,,0.039,149,0 +1,chat,small_deep,0.571,,,,0.035,149,0 +1,chat,small_deep,0.564,,,,0.034,149,0 +1,chat,small_deep,0.652,,,,0.034,149,0 +1,chat,small_deep,0.711,,,,0.051,149,0 +1,chat,small_deep,0.603,,,,0.048,149,0 +1,chat,small_deep,0.588,,,,0.041,149,0 +1,chat,small_deep,0.73,,,,0.037,149,0 +1,chat,small_deep,0.651,,,,0.044,149,0 +1,composite,insert_100000,1289.336,1513.1839999230579,45729460,6856704,,, +1,composite,hot_first,0.076,,,,0.228,5,0 +1,composite,hot_first,0.037,,,,0.039,5,0 +1,composite,hot_first,0.027,,,,0.03,5,0 +1,composite,hot_first,0.033,,,,0.039,5,0 +1,composite,hot_first,0.031,,,,0.039,5,0 +1,composite,hot_first,0.032,,,,0.039,5,0 +1,composite,hot_first,0.049,,,,0.092,5,0 +1,composite,hot_first,0.048,,,,0.04,5,0 +1,composite,hot_first,0.033,,,,0.051,5,0 +1,composite,hot_first,0.042,,,,0.051,5,0 +1,composite,hot_first,0.033,,,,0.038,5,0 +1,composite,hot_first,0.033,,,,0.107,5,0 +1,composite,hot_first,0.031,,,,0.038,5,0 +1,composite,hot_first,0.032,,,,0.039,5,0 +1,composite,hot_first,0.032,,,,0.038,5,0 +1,composite,hot_first,0.032,,,,0.041,5,0 +1,composite,hot_first,0.034,,,,0.054,5,0 +1,composite,hot_first,0.032,,,,0.039,5,0 +1,composite,hot_first,0.031,,,,0.038,5,0 +1,composite,hot_first,0.034,,,,0.049,5,0 +1,composite,hot_deep,0.084,,,,0.25,6,0 +1,composite,hot_deep,0.038,,,,0.039,6,0 +1,composite,hot_deep,0.034,,,,0.061,6,0 +1,composite,hot_deep,0.032,,,,0.039,6,0 +1,composite,hot_deep,0.073,,,,0.04,6,0 +1,composite,hot_deep,0.033,,,,0.042,6,0 +1,composite,hot_deep,0.043,,,,0.039,6,0 +1,composite,hot_deep,0.035,,,,0.042,6,0 +1,composite,hot_deep,0.034,,,,0.043,6,0 +1,composite,hot_deep,0.034,,,,0.04,6,0 +1,composite,hot_deep,0.033,,,,0.04,6,0 +1,composite,hot_deep,0.035,,,,0.056,6,0 +1,composite,hot_deep,0.031,,,,0.032,6,0 +1,composite,hot_deep,0.029,,,,0.029,6,0 +1,composite,hot_deep,0.043,,,,0.05,6,0 +1,composite,hot_deep,0.047,,,,0.054,6,0 +1,composite,hot_deep,0.048,,,,0.045,6,0 +1,composite,hot_deep,0.047,,,,0.097,6,0 +1,composite,hot_deep,0.04,,,,0.059,6,0 +1,composite,hot_deep,0.045,,,,0.095,6,0 +1,composite,small_deep,0.462,,,,0.246,50,0 +1,composite,small_deep,0.375,,,,0.042,44,0 +1,composite,small_deep,0.349,,,,0.042,44,0 +1,composite,small_deep,0.345,,,,0.047,44,0 +1,composite,small_deep,0.338,,,,0.04,44,0 +1,composite,small_deep,0.412,,,,0.043,44,0 +1,composite,small_deep,0.333,,,,0.044,44,0 +1,composite,small_deep,0.325,,,,0.036,44,0 +1,composite,small_deep,0.339,,,,0.045,44,0 +1,composite,small_deep,0.339,,,,0.05,44,0 +1,composite,small_deep,0.336,,,,0.045,44,0 +1,composite,small_deep,0.358,,,,0.051,44,0 +1,composite,small_deep,0.369,,,,0.059,44,0 +1,composite,small_deep,0.389,,,,0.057,44,0 +1,composite,small_deep,0.367,,,,0.055,44,0 +1,composite,small_deep,0.352,,,,0.054,44,0 +1,composite,small_deep,0.335,,,,0.05,44,0 +1,composite,small_deep,0.393,,,,0.031,44,0 +1,composite,small_deep,0.329,,,,0.031,44,0 +1,composite,small_deep,0.397,,,,0.028,44,0 +2,composite,insert_100000,1165.973,1402.5969000067562,45729460,6856704,,, +2,composite,hot_first,0.079,,,,0.266,5,0 +2,composite,hot_first,0.042,,,,0.044,5,0 +2,composite,hot_first,0.047,,,,0.044,5,0 +2,composite,hot_first,0.05,,,,0.054,5,0 +2,composite,hot_first,0.046,,,,0.053,5,0 +2,composite,hot_first,0.047,,,,0.047,5,0 +2,composite,hot_first,0.087,,,,0.061,5,0 +2,composite,hot_first,0.032,,,,0.045,5,0 +2,composite,hot_first,0.034,,,,0.045,5,0 +2,composite,hot_first,0.034,,,,0.047,5,0 +2,composite,hot_first,0.047,,,,0.039,5,0 +2,composite,hot_first,0.031,,,,0.04,5,0 +2,composite,hot_first,0.059,,,,0.041,5,0 +2,composite,hot_first,0.029,,,,0.036,5,0 +2,composite,hot_first,0.058,,,,0.03,5,0 +2,composite,hot_first,0.033,,,,0.042,5,0 +2,composite,hot_first,0.063,,,,0.04,5,0 +2,composite,hot_first,0.032,,,,0.039,5,0 +2,composite,hot_first,0.032,,,,0.039,5,0 +2,composite,hot_first,0.032,,,,0.039,5,0 +2,composite,hot_deep,0.072,,,,0.232,6,0 +2,composite,hot_deep,0.049,,,,0.042,6,0 +2,composite,hot_deep,0.032,,,,0.039,6,0 +2,composite,hot_deep,0.032,,,,0.039,6,0 +2,composite,hot_deep,0.034,,,,0.05,6,0 +2,composite,hot_deep,0.041,,,,0.045,6,0 +2,composite,hot_deep,0.032,,,,0.038,6,0 +2,composite,hot_deep,0.034,,,,0.039,6,0 +2,composite,hot_deep,0.035,,,,0.054,6,0 +2,composite,hot_deep,0.033,,,,0.042,6,0 +2,composite,hot_deep,0.033,,,,0.04,6,0 +2,composite,hot_deep,0.032,,,,0.038,6,0 +2,composite,hot_deep,0.033,,,,0.044,6,0 +2,composite,hot_deep,0.033,,,,0.038,6,0 +2,composite,hot_deep,0.033,,,,0.047,6,0 +2,composite,hot_deep,0.037,,,,0.057,6,0 +2,composite,hot_deep,0.057,,,,0.043,6,0 +2,composite,hot_deep,0.038,,,,0.052,6,0 +2,composite,hot_deep,0.039,,,,0.054,6,0 +2,composite,hot_deep,0.037,,,,0.045,6,0 +2,composite,small_deep,0.432,,,,0.25,50,0 +2,composite,small_deep,0.388,,,,0.04,44,0 +2,composite,small_deep,0.334,,,,0.057,44,0 +2,composite,small_deep,0.367,,,,0.045,44,0 +2,composite,small_deep,0.329,,,,0.043,44,0 +2,composite,small_deep,0.395,,,,0.042,44,0 +2,composite,small_deep,0.353,,,,0.038,44,0 +2,composite,small_deep,0.325,,,,0.058,44,0 +2,composite,small_deep,0.328,,,,0.04,44,0 +2,composite,small_deep,0.343,,,,0.043,44,0 +2,composite,small_deep,0.351,,,,0.036,44,0 +2,composite,small_deep,0.382,,,,0.033,44,0 +2,composite,small_deep,0.425,,,,0.05,44,0 +2,composite,small_deep,0.348,,,,0.059,44,0 +2,composite,small_deep,0.337,,,,0.049,44,0 +2,composite,small_deep,0.349,,,,0.04,44,0 +2,composite,small_deep,0.365,,,,0.041,44,0 +2,composite,small_deep,0.357,,,,0.036,44,0 +2,composite,small_deep,0.389,,,,0.032,44,0 +2,composite,small_deep,0.349,,,,0.034,44,0 +2,chat,insert_100000,1018.697,1253.4370000939816,41896134,663552,,, +2,chat,hot_first,25.791,,,,0.228,2862,0 +2,chat,hot_first,23.947,,,,0.067,2856,0 +2,chat,hot_first,25.473,,,,0.066,2856,0 +2,chat,hot_first,23.76,,,,0.068,2856,0 +2,chat,hot_first,24.276,,,,0.067,2856,0 +2,chat,hot_first,24.098,,,,0.069,2856,0 +2,chat,hot_first,23.935,,,,0.131,2856,0 +2,chat,hot_first,24.238,,,,0.27,2856,0 +2,chat,hot_first,25.161,,,,0.067,2856,0 +2,chat,hot_first,25.352,,,,0.067,2856,0 +2,chat,hot_first,24.303,,,,0.128,2856,0 +2,chat,hot_first,26.555,,,,0.091,2856,0 +2,chat,hot_first,26.665,,,,0.095,2856,0 +2,chat,hot_first,25.122,,,,0.069,2856,0 +2,chat,hot_first,24.817,,,,0.068,2856,0 +2,chat,hot_first,25.286,,,,0.096,2856,0 +2,chat,hot_first,26.137,,,,0.07,2856,0 +2,chat,hot_first,28.647,,,,0.069,2856,0 +2,chat,hot_first,25.666,,,,0.078,2856,0 +2,chat,hot_first,26.139,,,,0.069,2856,0 +2,chat,hot_deep,9.254,,,,0.297,2862,0 +2,chat,hot_deep,8.531,,,,0.066,2856,0 +2,chat,hot_deep,8.205,,,,0.056,2856,0 +2,chat,hot_deep,8.261,,,,0.076,2856,0 +2,chat,hot_deep,8.88,,,,0.057,2856,0 +2,chat,hot_deep,9.532,,,,0.067,2856,0 +2,chat,hot_deep,8.369,,,,0.069,2856,0 +2,chat,hot_deep,8.19,,,,0.06,2856,0 +2,chat,hot_deep,8.892,,,,0.068,2856,0 +2,chat,hot_deep,7.627,,,,0.058,2856,0 +2,chat,hot_deep,9.503,,,,0.065,2856,0 +2,chat,hot_deep,8.499,,,,0.062,2856,0 +2,chat,hot_deep,8.671,,,,0.066,2856,0 +2,chat,hot_deep,8.32,,,,0.058,2856,0 +2,chat,hot_deep,8.058,,,,0.065,2856,0 +2,chat,hot_deep,8.972,,,,0.061,2856,0 +2,chat,hot_deep,9.806,,,,0.075,2856,0 +2,chat,hot_deep,9.06,,,,0.058,2856,0 +2,chat,hot_deep,8.438,,,,0.056,2856,0 +2,chat,hot_deep,8.636,,,,0.071,2856,0 +2,chat,small_deep,0.945,,,,0.227,155,0 +2,chat,small_deep,0.73,,,,0.045,149,0 +2,chat,small_deep,0.591,,,,0.045,149,0 +2,chat,small_deep,0.716,,,,0.043,149,0 +2,chat,small_deep,0.736,,,,0.057,149,0 +2,chat,small_deep,0.682,,,,0.052,149,0 +2,chat,small_deep,0.622,,,,0.04,149,0 +2,chat,small_deep,0.602,,,,0.034,149,0 +2,chat,small_deep,0.657,,,,0.029,149,0 +2,chat,small_deep,0.587,,,,0.039,149,0 +2,chat,small_deep,0.752,,,,0.037,149,0 +2,chat,small_deep,0.669,,,,0.079,149,0 +2,chat,small_deep,0.671,,,,0.053,149,0 +2,chat,small_deep,0.646,,,,0.041,149,0 +2,chat,small_deep,0.727,,,,0.032,149,0 +2,chat,small_deep,0.659,,,,0.064,149,0 +2,chat,small_deep,0.66,,,,0.063,149,0 +2,chat,small_deep,0.642,,,,0.048,149,0 +2,chat,small_deep,0.612,,,,0.042,149,0 +2,chat,small_deep,0.585,,,,0.042,149,0 +3,chat,insert_100000,1071.655,1292.7609999896958,41896134,663552,,, +3,chat,hot_first,26.495,,,,0.265,2862,0 +3,chat,hot_first,27.361,,,,0.069,2856,0 +3,chat,hot_first,24.395,,,,0.231,2856,0 +3,chat,hot_first,25.486,,,,0.072,2856,0 +3,chat,hot_first,25.114,,,,0.068,2856,0 +3,chat,hot_first,26.402,,,,0.068,2856,0 +3,chat,hot_first,25.725,,,,0.081,2856,0 +3,chat,hot_first,25.563,,,,0.083,2856,0 +3,chat,hot_first,24.829,,,,0.07,2856,0 +3,chat,hot_first,25.111,,,,0.067,2856,0 +3,chat,hot_first,24.88,,,,0.07,2856,0 +3,chat,hot_first,25.728,,,,0.078,2856,0 +3,chat,hot_first,26.729,,,,0.068,2856,0 +3,chat,hot_first,27.725,,,,0.069,2856,0 +3,chat,hot_first,28.529,,,,0.069,2856,0 +3,chat,hot_first,24.694,,,,0.069,2856,0 +3,chat,hot_first,25.886,,,,0.076,2856,0 +3,chat,hot_first,24.727,,,,0.079,2856,0 +3,chat,hot_first,24.851,,,,0.069,2856,0 +3,chat,hot_first,31.14,,,,0.067,2856,0 +3,chat,hot_deep,14.786,,,,0.26,2862,0 +3,chat,hot_deep,9.625,,,,0.216,2856,0 +3,chat,hot_deep,9.365,,,,0.136,2856,0 +3,chat,hot_deep,8.736,,,,0.069,2856,0 +3,chat,hot_deep,8.199,,,,0.078,2856,0 +3,chat,hot_deep,7.894,,,,0.062,2856,0 +3,chat,hot_deep,7.94,,,,0.056,2856,0 +3,chat,hot_deep,7.95,,,,0.062,2856,0 +3,chat,hot_deep,8.322,,,,0.091,2856,0 +3,chat,hot_deep,8.22,,,,0.069,2856,0 +3,chat,hot_deep,7.937,,,,0.069,2856,0 +3,chat,hot_deep,8.127,,,,0.069,2856,0 +3,chat,hot_deep,8.05,,,,0.066,2856,0 +3,chat,hot_deep,9.153,,,,0.086,2856,0 +3,chat,hot_deep,9.807,,,,0.078,2856,0 +3,chat,hot_deep,8.263,,,,0.081,2856,0 +3,chat,hot_deep,8.094,,,,0.057,2856,0 +3,chat,hot_deep,9.194,,,,0.061,2856,0 +3,chat,hot_deep,8.47,,,,0.054,2856,0 +3,chat,hot_deep,7.938,,,,0.068,2856,0 +3,chat,small_deep,0.854,,,,0.275,155,0 +3,chat,small_deep,0.755,,,,0.042,149,0 +3,chat,small_deep,0.654,,,,0.041,149,0 +3,chat,small_deep,0.633,,,,0.042,149,0 +3,chat,small_deep,0.634,,,,0.042,149,0 +3,chat,small_deep,0.619,,,,0.039,149,0 +3,chat,small_deep,0.754,,,,0.04,149,0 +3,chat,small_deep,0.654,,,,0.058,149,0 +3,chat,small_deep,0.709,,,,0.058,149,0 +3,chat,small_deep,0.697,,,,0.052,149,0 +3,chat,small_deep,0.611,,,,0.075,149,0 +3,chat,small_deep,0.73,,,,0.052,149,0 +3,chat,small_deep,0.702,,,,0.057,149,0 +3,chat,small_deep,0.734,,,,0.063,149,0 +3,chat,small_deep,0.646,,,,0.046,149,0 +3,chat,small_deep,0.736,,,,0.038,149,0 +3,chat,small_deep,0.651,,,,0.055,149,0 +3,chat,small_deep,0.601,,,,0.043,149,0 +3,chat,small_deep,0.568,,,,0.034,149,0 +3,chat,small_deep,0.575,,,,0.035,149,0 +3,composite,insert_100000,1057.69,1288.085800013505,45729460,6856704,,, +3,composite,hot_first,0.069,,,,0.238,5,0 +3,composite,hot_first,0.043,,,,0.051,5,0 +3,composite,hot_first,0.045,,,,0.063,5,0 +3,composite,hot_first,0.037,,,,0.047,5,0 +3,composite,hot_first,0.034,,,,0.039,5,0 +3,composite,hot_first,0.041,,,,0.047,5,0 +3,composite,hot_first,0.047,,,,0.044,5,0 +3,composite,hot_first,0.034,,,,0.047,5,0 +3,composite,hot_first,0.046,,,,0.046,5,0 +3,composite,hot_first,0.036,,,,0.055,5,0 +3,composite,hot_first,0.046,,,,0.051,5,0 +3,composite,hot_first,0.064,,,,0.061,5,0 +3,composite,hot_first,0.039,,,,0.056,5,0 +3,composite,hot_first,0.038,,,,0.05,5,0 +3,composite,hot_first,0.051,,,,0.058,5,0 +3,composite,hot_first,0.035,,,,0.054,5,0 +3,composite,hot_first,0.038,,,,0.061,5,0 +3,composite,hot_first,0.034,,,,0.047,5,0 +3,composite,hot_first,0.036,,,,0.049,5,0 +3,composite,hot_first,0.037,,,,0.064,5,0 +3,composite,hot_deep,0.074,,,,0.235,6,0 +3,composite,hot_deep,0.044,,,,0.045,6,0 +3,composite,hot_deep,0.036,,,,0.041,6,0 +3,composite,hot_deep,0.034,,,,0.043,6,0 +3,composite,hot_deep,0.034,,,,0.041,6,0 +3,composite,hot_deep,0.035,,,,0.047,6,0 +3,composite,hot_deep,0.036,,,,0.047,6,0 +3,composite,hot_deep,0.035,,,,0.041,6,0 +3,composite,hot_deep,0.035,,,,0.043,6,0 +3,composite,hot_deep,0.035,,,,0.044,6,0 +3,composite,hot_deep,0.048,,,,0.044,6,0 +3,composite,hot_deep,0.036,,,,0.057,6,0 +3,composite,hot_deep,0.034,,,,0.042,6,0 +3,composite,hot_deep,0.035,,,,0.043,6,0 +3,composite,hot_deep,0.034,,,,0.04,6,0 +3,composite,hot_deep,0.036,,,,0.041,6,0 +3,composite,hot_deep,0.052,,,,0.056,6,0 +3,composite,hot_deep,0.037,,,,0.049,6,0 +3,composite,hot_deep,0.034,,,,0.041,6,0 +3,composite,hot_deep,0.033,,,,0.038,6,0 +3,composite,small_deep,0.459,,,,0.265,50,0 +3,composite,small_deep,0.611,,,,0.06,44,0 +3,composite,small_deep,0.656,,,,0.139,44,0 +3,composite,small_deep,0.637,,,,0.112,44,0 +3,composite,small_deep,0.653,,,,0.122,44,0 +3,composite,small_deep,0.665,,,,0.114,44,0 +3,composite,small_deep,0.67,,,,0.111,44,0 +3,composite,small_deep,0.854,,,,0.112,44,0 +3,composite,small_deep,0.447,,,,0.092,44,0 +3,composite,small_deep,0.641,,,,0.083,44,0 +3,composite,small_deep,0.412,,,,0.08,44,0 +3,composite,small_deep,0.971,,,,0.078,44,0 +3,composite,small_deep,0.509,,,,0.079,44,0 +3,composite,small_deep,0.627,,,,0.09,44,0 +3,composite,small_deep,0.554,,,,0.087,44,0 +3,composite,small_deep,0.436,,,,0.097,44,0 +3,composite,small_deep,0.442,,,,0.083,44,0 +3,composite,small_deep,0.376,,,,0.063,44,0 +3,composite,small_deep,0.63,,,,0.083,44,0 +3,composite,small_deep,0.409,,,,0.07,44,0 diff --git a/evidence/research.json b/evidence/research.json new file mode 100644 index 0000000..7ae51e9 --- /dev/null +++ b/evidence/research.json @@ -0,0 +1,199 @@ +{ + "utc": "2026-10-16T12:07:09.786905+00:00", + "image": "postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "host": "linux/x86_64 CPUs=12 MemoryBytes=3956064256", + "limits": { + "cpus_per_container": 2, + "memory_bytes": 536870912, + "clients": 1 + }, + "method": { + "repeats": 3, + "read_samples_per_case_per_repeat": 20, + "warmup": 3, + "order": [ + "chat,composite", + "composite,chat", + "chat,composite" + ], + "distribution": [ + 80000, + 15000, + 5000 + ], + "parallel_query": false, + "jit": false + }, + "postgres": "PostgreSQL 17.6 (Debian 17.6-2.pgdg12+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", + "summary": [ + { + "variant": "chat", + "case": "insert_100000", + "samples": 3, + "median_ms": 1071.655, + "p95_ms": null, + "p99_ms": null, + "index_bytes": 663552, + "wal_bytes": 41896134, + "shared_hit_blocks": null, + "shared_read_blocks": null + }, + { + "variant": "chat", + "case": "hot_first", + "samples": 60, + "median_ms": 25.392000000000003, + "p95_ms": 27.725, + "p99_ms": 31.14, + "index_bytes": null, + "wal_bytes": null, + "shared_hit_blocks": 2856.0, + "shared_read_blocks": 0.0 + }, + { + "variant": "chat", + "case": "hot_deep", + "samples": 60, + "median_ms": 8.729, + "p95_ms": 9.807, + "p99_ms": 14.786, + "index_bytes": null, + "wal_bytes": null, + "shared_hit_blocks": 2856.0, + "shared_read_blocks": 0.0 + }, + { + "variant": "chat", + "case": "small_deep", + "samples": 60, + "median_ms": 0.6515, + "p95_ms": 0.755, + "p99_ms": 0.945, + "index_bytes": null, + "wal_bytes": null, + "shared_hit_blocks": 149.0, + "shared_read_blocks": 0.0 + }, + { + "variant": "composite", + "case": "insert_100000", + "samples": 3, + "median_ms": 1165.973, + "p95_ms": null, + "p99_ms": null, + "index_bytes": 6856704, + "wal_bytes": 45729460, + "shared_hit_blocks": null, + "shared_read_blocks": null + }, + { + "variant": "composite", + "case": "hot_first", + "samples": 60, + "median_ms": 0.0365, + "p95_ms": 0.069, + "p99_ms": 0.087, + "index_bytes": null, + "wal_bytes": null, + "shared_hit_blocks": 5.0, + "shared_read_blocks": 0.0 + }, + { + "variant": "composite", + "case": "hot_deep", + "samples": 60, + "median_ms": 0.035, + "p95_ms": 0.072, + "p99_ms": 0.084, + "index_bytes": null, + "wal_bytes": null, + "shared_hit_blocks": 6.0, + "shared_read_blocks": 0.0 + }, + { + "variant": "composite", + "case": "small_deep", + "samples": 60, + "median_ms": 0.379, + "p95_ms": 0.665, + "p99_ms": 0.971, + "index_bytes": null, + "wal_bytes": null, + "shared_hit_blocks": 44.0, + "shared_read_blocks": 0.0 + } + ], + "restore": { + "source_container": "lab2-research-83fe774600", + "target_container": "lab2-restore-83fe774600", + "source_volume": "lab2-research-83fe774600-data", + "new_volume": "lab2-restore-83fe774600-data", + "destination_initial_public_tables": 0, + "dump_seconds": 0.5826623999746516, + "dump_bytes": 3027977, + "dump_path": "backups/research-83fe774600.dump", + "destination_startup_seconds": 2.4172622999176383, + "pg_restore_seconds": 1.1627591999713331, + "recovery_through_verification_seconds": 6.650994199910201, + "before_backup": { + "counts": { + "users": 1, + "chats": 3, + "chat_members": 3, + "messages": 100000, + "schema_migrations": 2 + }, + "chat_counts": { + "6adca606-cb42-ce80-9dfe-e9c5ab3f1b51": 80000, + "0edbe356-1e20-7355-3db8-96e0eb8b4f65": 5000, + "64774a49-48b0-211f-0ac6-28d7a0db260f": 15000 + }, + "history_sha256": "77d6cd55c7467440c0c500f3aa8fcf74145fbd6b108b347cea51d54c5886e52a", + "samples": { + "200726dc-60e7-4926-cb2a-83366f5e1508": "0035c2ed1e41b01413e82ed95631c6391168b910dd6e78a6a6e8a4c595fea1dd", + "2ec90eaf-63d3-874f-03e0-4b72e302aa34": "40e53c4b3479d711e9ccf09ba951fc67a00083274b65fee7723c10a6bce1bbeb", + "5250e7db-fada-8da9-5526-2558f592e527": "640901647462319a187ea1d1277f54880bf9c4df1ba69621a25190f22e7b21f2", + "59f8c672-0c1a-ed3e-9fbc-bbe8c574a7a2": "f2a2c79fe54650df021d8e3b752e221d481002a9715452fad82dc579fe0c87ee", + "7a15b473-994f-e836-a01a-4ff3eafcb285": "7ce58b566161b6dfcbb418c0abe90574c3eae47d98378969c6c12adfdf7b3fb6", + "7aaf2bd4-084d-2b18-d619-5cf0cfecd280": "865bcb893c76498eb31c7f06d19db18db520771824b8157e9a561e5dbb9d1d0a", + "80408f00-16cd-9fd0-de25-77a69236c5f5": "b70b5407f08ab2bbea71b530b9803b2aece580985681408f1acffa96a4df45e2", + "8dbc017a-9f4f-1a71-ae0f-6e1e4926fa09": "0fd7cdf68fe068c441a57ae369997050ed584167a7988d21e9131dc593f9ba89", + "ae678afe-42a2-1c39-aa5d-83256f57f1a7": "6a305999f64702e8f8475797e0884ff473c271c42f56ded6e51546563591c3b6", + "b4d94967-7ae4-b21c-60b7-ffd32fffd17c": "04e7ea69116281e36e3fbe9e753e81c7585dd24f01c7f8245c4a388c8326a368", + "cf53958f-c8bf-496e-e9ab-d1c4385382ea": "b6796018ff49584e48afe5ce4c5d360b018f30f205eed9073f44f8504ae091da" + } + }, + "after_restore": { + "counts": { + "users": 1, + "chats": 3, + "chat_members": 3, + "messages": 100000, + "schema_migrations": 2 + }, + "chat_counts": { + "64774a49-48b0-211f-0ac6-28d7a0db260f": 15000, + "6adca606-cb42-ce80-9dfe-e9c5ab3f1b51": 80000, + "0edbe356-1e20-7355-3db8-96e0eb8b4f65": 5000 + }, + "history_sha256": "77d6cd55c7467440c0c500f3aa8fcf74145fbd6b108b347cea51d54c5886e52a", + "samples": { + "200726dc-60e7-4926-cb2a-83366f5e1508": "0035c2ed1e41b01413e82ed95631c6391168b910dd6e78a6a6e8a4c595fea1dd", + "2ec90eaf-63d3-874f-03e0-4b72e302aa34": "40e53c4b3479d711e9ccf09ba951fc67a00083274b65fee7723c10a6bce1bbeb", + "5250e7db-fada-8da9-5526-2558f592e527": "640901647462319a187ea1d1277f54880bf9c4df1ba69621a25190f22e7b21f2", + "59f8c672-0c1a-ed3e-9fbc-bbe8c574a7a2": "f2a2c79fe54650df021d8e3b752e221d481002a9715452fad82dc579fe0c87ee", + "7a15b473-994f-e836-a01a-4ff3eafcb285": "7ce58b566161b6dfcbb418c0abe90574c3eae47d98378969c6c12adfdf7b3fb6", + "7aaf2bd4-084d-2b18-d619-5cf0cfecd280": "865bcb893c76498eb31c7f06d19db18db520771824b8157e9a561e5dbb9d1d0a", + "80408f00-16cd-9fd0-de25-77a69236c5f5": "b70b5407f08ab2bbea71b530b9803b2aece580985681408f1acffa96a4df45e2", + "8dbc017a-9f4f-1a71-ae0f-6e1e4926fa09": "0fd7cdf68fe068c441a57ae369997050ed584167a7988d21e9131dc593f9ba89", + "ae678afe-42a2-1c39-aa5d-83256f57f1a7": "6a305999f64702e8f8475797e0884ff473c271c42f56ded6e51546563591c3b6", + "b4d94967-7ae4-b21c-60b7-ffd32fffd17c": "04e7ea69116281e36e3fbe9e753e81c7585dd24f01c7f8245c4a388c8326a368", + "cf53958f-c8bf-496e-e9ab-d1c4385382ea": "b6796018ff49584e48afe5ce4c5d360b018f30f205eed9073f44f8504ae091da" + } + }, + "source_count_after_backup": 100001, + "restored_count": 100000, + "post_backup_marker_absent": true + }, + "result": "PASS: two indexes, 100000 uneven messages, write cost, new-volume restore and SHA256 verified" +} diff --git a/evidence/restart-after.txt b/evidence/restart-after.txt new file mode 100644 index 0000000..a828d25 --- /dev/null +++ b/evidence/restart-after.txt @@ -0,0 +1,3 @@ +/mtusi-lab2-api-1 2026-10-16T12:12:56.423734297Z +/mtusi-lab2-db-1 2026-10-16T12:12:52.943113207Z +/mtusi-lab2-valkey-1 2026-10-16T12:12:52.868740837Z diff --git a/evidence/restart-before.txt b/evidence/restart-before.txt new file mode 100644 index 0000000..01bca4a --- /dev/null +++ b/evidence/restart-before.txt @@ -0,0 +1,3 @@ +/mtusi-lab2-api-1 2026-10-16T11:39:33.975833867Z +/mtusi-lab2-db-1 2026-10-16T11:39:30.617236669Z +/mtusi-lab2-valkey-1 2026-10-16T11:39:30.667089298Z diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..1e09963 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,5 @@ +2026-10-16T11:21:04Z +$ make test BASE_URL=http://127.0.0.1:8080 +python3 tests/smoke.py --base-url "http://127.0.0.1:8080" +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +exit_code=0 diff --git a/evidence/source-check.txt b/evidence/source-check.txt new file mode 100644 index 0000000..794a0c0 --- /dev/null +++ b/evidence/source-check.txt @@ -0,0 +1,4 @@ +2026-10-16T11:21:22Z +$ python -X utf8 scripts/verify_sources.py +PASS: 17 mandatory files unchanged; public Makefile targets preserved +exit_code=0 diff --git a/evidence/source-manifest.json b/evidence/source-manifest.json new file mode 100644 index 0000000..c3a90c6 --- /dev/null +++ b/evidence/source-manifest.json @@ -0,0 +1,74 @@ +{ + "baseline": "65003b61195361aac238f19c9bff0027ca58e09e", + "files": [ + { + "path": "CONTRIBUTING.md", + "sha256_lf": "2c01dd698cc9653dece49a94c9fd3f58d36e84764aaa643b46e26d7e599dc12e" + }, + { + "path": "COURSE.md", + "sha256_lf": "c34d53cd082d194aa4b58f504c4e0348d25b6afec3d6a9c0963dd01c9c81e971" + }, + { + "path": "MAINTAINERS.md", + "sha256_lf": "59a70ede89ba308a782f0695f8441dfa4b75479d155966303ab86addc36f8d1f" + }, + { + "path": "README.md", + "sha256_lf": "4f14a97911bd3ea335868d37fec187d1738b42f91a7d4780d33b721edf25b6dc" + }, + { + "path": "contracts/README.md", + "sha256_lf": "72df27a559f88ffa3fd8c3c79d3c32f6db2c5986a0602677c51994c434a3fc4f" + }, + { + "path": "contracts/examples.json", + "sha256_lf": "0a71cec72903af556df85f939c7e347fc73a86a1a4ffc39fb9c8ded169b7df45" + }, + { + "path": "contracts/infrastructure.json", + "sha256_lf": "7efd8b1e6023ddc3ff9747d4f663ee772c1293b782bb9e74bd1c3fbcca9c68d8" + }, + { + "path": "contracts/openapi.json", + "sha256_lf": "cff3aa6b34855b573401b0ac393fdfe327b6a12da51f315fbbd67168ce3762c4" + }, + { + "path": "lab.json", + "sha256_lf": "d63eac8df585a63153cff0c632e6b81b2088316b53d531eecc856ec1da7855e4" + }, + { + "path": "requirements-dev.txt", + "sha256_lf": "4cefdb67007caf026d1613be7c519c01b4c8343db7c4f4b618677e01cf113525" + }, + { + "path": "scripts/check_contract.py", + "sha256_lf": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671" + }, + { + "path": "scripts/validate_openapi.py", + "sha256_lf": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54" + }, + { + "path": "tests/README.md", + "sha256_lf": "e0bfa7e196797f6a5bf6e44d8ad2ffeb82651927ff5d8484eb1c2dda1ec7dec5" + }, + { + "path": "tests/client.py", + "sha256_lf": "8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d" + }, + { + "path": "tests/load.py", + "sha256_lf": "6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933" + }, + { + "path": "tests/resilience.py", + "sha256_lf": "1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b" + }, + { + "path": "tests/smoke.py", + "sha256_lf": "52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db" + } + ], + "result": "PASS: mandatory files identical (LF normalized); original Makefile targets preserved" +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c7a36b7 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module mtusi-messenger + +go 1.26.0 + +require ( + github.com/jackc/pgx/v5 v5.7.6 + github.com/redis/go-redis/v9 v9.12.1 + golang.org/x/crypto v0.43.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/text v0.30.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..18d327f --- /dev/null +++ b/go.sum @@ -0,0 +1,40 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= +github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..2687e1b --- /dev/null +++ b/main.go @@ -0,0 +1,360 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +type App struct { + db *pgxpool.Pool + sessions *redis.Client + secret []byte + ttl time.Duration + slots, kdf chan struct{} + stopping atomic.Bool + dummyHash string +} + +func env(key, value string) string { + if v := os.Getenv(key); v != "" { + return v + } + return value +} +func positive(key string, fallback int) int { + n, e := strconv.Atoi(env(key, strconv.Itoa(fallback))) + if e != nil || n <= 0 { + panic("invalid positive configuration: " + key) + } + return n +} +func secretFile(key string) (string, error) { + b, e := os.ReadFile(os.Getenv(key)) + if e != nil { + return "", fmt.Errorf("cannot read %s", key) + } + s := strings.TrimSpace(string(b)) + if s == "" { + return "", fmt.Errorf("empty %s", key) + } + return s, nil +} + +func (a *App) dispatch(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, any, error) { + path := r.URL.Path + if r.Method == "GET" && (path == "/health/live" || path == "/health/ready") { + if path == "/health/ready" && (a.stopping.Load() || a.db.Ping(ctx) != nil || a.sessions.Ping(ctx).Err() != nil) { + return 0, nil, unavailable() + } + return 200, map[string]string{"status": "ok"}, nil + } + if a.stopping.Load() { + return 0, nil, unavailable() + } + if r.Method == "POST" && path == "/api/v1/auth/sessions" { + return a.login(ctx, r) + } + if r.Method == "POST" && path == "/api/v1/users" { + body, err := readObject(w, r, 128*1024, []string{"username", "display_name", "password"}) + if err != nil { + return 0, nil, err + } + name, err := textField(body, "username", 32) + if err != nil || !usernamePattern.MatchString(name) { + return 0, nil, invalid() + } + display, err := textField(body, "display_name", 100) + if err != nil { + return 0, nil, err + } + p, err := password(body) + if err != nil { + return 0, nil, err + } + if err = a.kdfSlot(ctx); err != nil { + return 0, nil, err + } + hash := hashPassword(p) + <-a.kdf + u, err := scanUser(a.db.QueryRow(ctx, "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) RETURNING id::text,username,display_name,created_at", uuid(), name, []byte(display), hash)) + return 201, u, err + } + user, key, err := a.authenticate(ctx, r) + if err != nil { + return 0, nil, err + } + if r.Method == "DELETE" && path == "/api/v1/auth/sessions/current" { + count, e := a.sessions.Del(ctx, key).Result() + if e != nil { + return 0, nil, unavailable() + } + if count == 0 { + return 0, nil, unauthorized() + } + return 204, nil, nil + } + if r.Method == "GET" && path == "/api/v1/users/me" { + u, e := a.user(ctx, user) + return 200, u, e + } + if path == "/api/v1/chats" { + if r.Method == "GET" { + scope := "chats:" + user + limit, c, e := a.pagination(r, scope) + if e != nil { + return 0, nil, e + } + page, e := a.listChats(ctx, user, limit, c, scope) + return 200, page, e + } + if r.Method == "POST" { + body, e := readObject(w, r, 128*1024, []string{"title", "member_ids"}) + if e != nil { + return 0, nil, e + } + title, e := textField(body, "title", 100) + if e != nil { + return 0, nil, e + } + var ids []string + if json.Unmarshal(body["member_ids"], &ids) != nil || len(ids) < 1 || len(ids) > 100 { + return 0, nil, invalid() + } + seen := map[string]bool{} + for i, id := range ids { + normalized, e := normalizeID(id) + if e != nil || seen[normalized] { + return 0, nil, invalid() + } + ids[i] = normalized + seen[normalized] = true + } + if !seen[user] { + return 0, nil, invalid() + } + chat, e := a.createChat(ctx, title, ids) + return 201, chat, e + } + } + if strings.HasPrefix(path, "/api/v1/chats/") { + parts := strings.Split(strings.TrimPrefix(path, "/api/v1/chats/"), "/") + id, e := normalizeID(parts[0]) + if e != nil { + return 0, nil, e + } + if len(parts) > 2 || (len(parts) == 2 && parts[1] != "messages") { + return 0, nil, missing() + } + chat, e := a.chat(ctx, id, user) + if e != nil { + return 0, nil, e + } + if len(parts) == 1 && r.Method == "GET" { + return 200, chat, nil + } + if len(parts) == 2 { + if r.Method == "GET" { + scope := "messages:" + id + limit, c, e := a.pagination(r, scope) + if e != nil { + return 0, nil, e + } + page, e := a.history(ctx, id, limit, c, scope) + return 200, page, e + } + if r.Method == "POST" { + body, e := readObject(w, r, 128*1024, []string{"text"}) + if e != nil { + return 0, nil, e + } + text, e := textField(body, "text", 4000) + if e != nil { + return 0, nil, e + } + m := Message{ID: uuid(), ChatID: id, SenderID: user, Text: text} + var created time.Time + e = a.db.QueryRow(ctx, "INSERT INTO messages(id,chat_id,sender_id,text) VALUES($1,$2,$3,$4) RETURNING created_at", m.ID, id, user, []byte(text)).Scan(&created) + m.CreatedAt = stamp(created) + return 201, m, dbError(e) + } + } + } + return 0, nil, missing() +} + +func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + requestID := uuid() + status := 500 + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Request-Id", requestID) + fail := func(err error) { + var e apiError + if !errors.As(err, &e) { + e = unavailable() + } + status = e.status + if status == 401 { + scheme := `Bearer realm="messenger"` + if r.URL.Path == "/api/v1/auth/sessions" { + scheme = `Basic realm="messenger", charset="UTF-8"` + } + w.Header().Set("WWW-Authenticate", scheme) + } + if status == 503 { + w.Header().Set("Retry-After", "1") + } + w.WriteHeader(status) + json.NewEncoder(w).Encode(map[string]any{"error": map[string]string{"code": e.code, "message": e.message, "request_id": requestID}}) + } + defer func() { + if recover() != nil { + fail(apiError{500, "internal_error", "Internal server error"}) + } + slog.Info("request", "method", r.Method, "status", status, "request_id", requestID, "duration_ms", time.Since(start).Milliseconds()) + }() + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + health := r.Method == "GET" && (r.URL.Path == "/health/live" || r.URL.Path == "/health/ready") + if !health { + select { + case a.slots <- struct{}{}: + defer func() { <-a.slots }() + default: + fail(unavailable()) + return + } + } + code, value, err := a.dispatch(ctx, w, r) + if err != nil { + fail(err) + return + } + status = code + w.WriteHeader(code) + if code != 204 { + json.NewEncoder(w).Encode(value) + } +} + +func run() error { + if env("SESSION_STORE", "valkey") != "valkey" { + return errors.New("SESSION_STORE must be valkey") + } + address := net.JoinHostPort(env("APP_HOST", "0.0.0.0"), env("APP_PORT", "8080")) + if len(os.Args) > 1 && os.Args[1] == "healthcheck" { + client := http.Client{Timeout: 2 * time.Second} + res, e := client.Get("http://127.0.0.1:" + env("APP_PORT", "8080") + "/health/ready") + if e != nil { + return errors.New("not ready") + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New("not ready") + } + return nil + } + dbURL, err := secretFile("DATABASE_URL_FILE") + if err != nil { + return err + } + cacheURL, err := secretFile("CACHE_URL_FILE") + if err != nil { + return err + } + signing, err := secretFile("CURSOR_SECRET_FILE") + if err != nil { + return err + } + if len(signing) < 32 { + return errors.New("cursor secret too short") + } + cfg, err := pgxpool.ParseConfig(dbURL) + if err != nil { + return errors.New("invalid database configuration") + } + cfg.MaxConns = int32(positive("DB_POOL_MAX", 8)) + cfg.MinConns = 0 + cfg.ConnConfig.ConnectTimeout = 2 * time.Second + cfg.ConnConfig.RuntimeParams["statement_timeout"] = "3000" + cfg.ConnConfig.RuntimeParams["lock_timeout"] = "1000" + cfg.ConnConfig.RuntimeParams["synchronous_commit"] = "on" + startup, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + db, err := pgxpool.NewWithConfig(startup, cfg) + if err != nil { + return errors.New("database setup failed") + } + defer db.Close() + if err = migrate(startup, db); err != nil { + return errors.New("migration failed") + } + if len(os.Args) > 1 && os.Args[1] == "migrate" { + slog.Info("migrations complete") + return nil + } + options, err := redis.ParseURL(cacheURL) + if err != nil { + return errors.New("invalid session store configuration") + } + options.PoolSize = 8 + options.MaxActiveConns = 8 + options.DialTimeout = time.Second + options.ReadTimeout = time.Second + options.WriteTimeout = time.Second + options.PoolTimeout = time.Second + options.MaxRetries = -1 + options.ContextTimeoutEnabled = true + sessions := redis.NewClient(options) + defer sessions.Close() + a := &App{db: db, sessions: sessions, secret: []byte(signing), ttl: time.Duration(positive("SESSION_TTL_SECONDS", 3600)) * time.Second, slots: make(chan struct{}, positive("MAX_INFLIGHT", 32)), kdf: make(chan struct{}, 2), dummyHash: hashPassword("unknown-user-dummy-password")} + server := &http.Server{Addr: address, Handler: a, ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 3 * time.Second, WriteTimeout: 8 * time.Second, IdleTimeout: 5 * time.Second, MaxHeaderBytes: 16 * 1024} + listener, err := net.Listen("tcp", address) + if err != nil { + return errors.New("listen failed") + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + done := make(chan struct{}) + go func() { + <-ctx.Done() + a.stopping.Store(true) + slog.Info("shutdown started") + shutdown, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + err := server.Shutdown(shutdown) + if err != nil { + server.Close() + } + slog.Info("shutdown complete", "forced", err != nil) + close(done) + }() + slog.Info("listening", "address", address, "environment", env("APP_ENV", "lab"), "session_store", "valkey", "session_ttl_seconds", int(a.ttl.Seconds())) + if err = server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + return errors.New("serve failed") + } + <-done + return nil +} +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + if err := run(); err != nil { + slog.Error("startup failed", "reason", err.Error()) + os.Exit(1) + } +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..69c00bb --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,28 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_page_idx ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); diff --git a/migrations/002_unicode_storage.sql b/migrations/002_unicode_storage.sql new file mode 100644 index 0000000..19a1643 --- /dev/null +++ b/migrations/002_unicode_storage.sql @@ -0,0 +1,12 @@ +-- PostgreSQL text cannot contain U+0000, although it is a valid JSON string +-- code point. Keep exact UTF-8 bytes for free-form API strings. The API checks +-- Unicode length/whitespace; these checks also bound stored byte lengths. +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name,'UTF8'); +ALTER TABLE users ADD CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title,'UTF8'); +ALTER TABLE chats ADD CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text,'UTF8'); +ALTER TABLE messages ADD CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/protocol.go b/protocol.go new file mode 100644 index 0000000..b5f5308 --- /dev/null +++ b/protocol.go @@ -0,0 +1,237 @@ +package main + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` +} +type Page[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} +type apiError struct { + status int + code, message string +} + +func (e apiError) Error() string { return e.code } +func invalid() apiError { return apiError{400, "invalid_request", "Invalid request"} } +func unavailable() apiError { return apiError{503, "unavailable", "Dependency unavailable"} } +func missing() apiError { return apiError{404, "not_found", "Object not found"} } +func unauthorized() apiError { return apiError{401, "unauthorized", "Valid Bearer session required"} } + +func uuid() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + panic(err) + } + b[6] = (b[6] & 15) | 64 + b[8] = (b[8] & 63) | 128 + s := hex.EncodeToString(b) + return s[:8] + "-" + s[8:12] + "-" + s[12:16] + "-" + s[16:20] + "-" + s[20:] +} + +var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func normalizeID(s string) (string, error) { + if !uuidPattern.MatchString(s) { + return "", invalid() + } + return strings.ToLower(s), nil +} + +func textField(m map[string]json.RawMessage, key string, max int) (string, error) { + var s string + b, ok := m[key] + if !ok || string(b) == "null" || json.Unmarshal(b, &s) != nil || strings.TrimSpace(s) == "" || utf8.RuneCountInString(s) > max { + return "", invalid() + } + return s, nil +} + +// Reject duplicate keys, invalid UTF-8, and unpaired UTF-16 escapes rather than +// letting encoding/json silently replace malformed Unicode with U+FFFD. +func validUnicode(b []byte) bool { + if !utf8.Valid(b) { + return false + } + for i := 0; i < len(b); i++ { + if b[i] != '\\' { + continue + } + i++ + if i >= len(b) { + return false + } + if b[i] != 'u' { + continue + } + if i+4 >= len(b) { + return false + } + n, e := strconv.ParseUint(string(b[i+1:i+5]), 16, 16) + if e != nil { + return false + } + i += 4 + if n >= 0xDC00 && n <= 0xDFFF { + return false + } + if n >= 0xD800 && n <= 0xDBFF { + if i+6 >= len(b) || string(b[i+1:i+3]) != `\u` { + return false + } + low, e := strconv.ParseUint(string(b[i+3:i+7]), 16, 16) + if e != nil || low < 0xDC00 || low > 0xDFFF { + return false + } + i += 6 + } + } + return true +} +func readObject(w http.ResponseWriter, r *http.Request, max int, fields []string) (map[string]json.RawMessage, error) { + media, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || media != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { + return nil, apiError{415, "unsupported_media_type", "Expected application/json UTF-8"} + } + b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, int64(max))) + if err != nil { + var large *http.MaxBytesError + if errors.As(err, &large) { + return nil, apiError{413, "payload_too_large", "Request body exceeds byte limit"} + } + return nil, invalid() + } + if !validUnicode(b) { + return nil, invalid() + } + d := json.NewDecoder(strings.NewReader(string(b))) + token, err := d.Token() + if err != nil || token != json.Delim('{') { + return nil, invalid() + } + m := map[string]json.RawMessage{} + for d.More() { + t, err := d.Token() + if err != nil { + return nil, invalid() + } + key, ok := t.(string) + if !ok { + return nil, invalid() + } + allowed := false + for _, f := range fields { + if key == f { + allowed = true + } + } + if !allowed || m[key] != nil { + return nil, invalid() + } + var value json.RawMessage + if d.Decode(&value) != nil { + return nil, invalid() + } + m[key] = value + } + if _, err = d.Token(); err != nil { + return nil, invalid() + } + if _, err = d.Token(); err != io.EOF { + return nil, invalid() + } + return m, nil +} + +type cursor struct { + Scope string `json:"s"` + Time string `json:"t"` + ID string `json:"i"` +} + +func (a *App) encodeCursor(scope, t, id string) string { + b, _ := json.Marshal(cursor{scope, t, id}) + h := hmac.New(sha256.New, a.secret) + h.Write(b) + return base64.RawURLEncoding.EncodeToString(b) + "." + base64.RawURLEncoding.EncodeToString(h.Sum(nil)) +} +func (a *App) pagination(r *http.Request, scope string) (int, cursor, error) { + values, e := url.ParseQuery(r.URL.RawQuery) + if e != nil { + return 0, cursor{}, invalid() + } + limit := 50 + if v, ok := values["limit"]; ok { + if len(v) != 1 || v[0] == "" { + return 0, cursor{}, invalid() + } + for _, c := range v[0] { + if c < '0' || c > '9' { + return 0, cursor{}, invalid() + } + } + limit, e = strconv.Atoi(v[0]) + if e != nil || limit < 1 || limit > 100 { + return 0, cursor{}, invalid() + } + } + c := cursor{} + if v, ok := values["cursor"]; ok { + if len(v) != 1 || len(v[0]) > 1024 { + return 0, c, invalid() + } + parts := strings.Split(v[0], ".") + if len(parts) != 2 { + return 0, c, invalid() + } + b, e := base64.RawURLEncoding.DecodeString(parts[0]) + if e != nil { + return 0, c, invalid() + } + sig, e := base64.RawURLEncoding.DecodeString(parts[1]) + if e != nil { + return 0, c, invalid() + } + h := hmac.New(sha256.New, a.secret) + h.Write(b) + if !hmac.Equal(sig, h.Sum(nil)) || json.Unmarshal(b, &c) != nil || c.Scope != scope { + return 0, c, invalid() + } + } + return limit, c, nil +} diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..a59deca --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,29 @@ +"""Create local secrets once. Never print values or overwrite existing secrets.""" +from pathlib import Path +import secrets +import shutil + +root = Path(__file__).resolve().parents[1] +folder = root / 'secrets' +folder.mkdir(exist_ok=True) +folder.chmod(0o700) +def put(name, value): + path = folder / name + if not path.exists(): + with path.open('x', encoding='utf-8', newline='\n') as f: + f.write(value + '\n') + # Parent directory is private; Compose mounts individual files for non-root services. + path.chmod(0o644) + return path.read_text(encoding='utf-8').strip() + +pg = put('postgres_password', secrets.token_hex(32)) +vk = put('valkey_password', secrets.token_hex(32)) +put('database_url', f'postgres://messenger:{pg}@db:5432/messenger?sslmode=disable') +put('cache_url', f'redis://:{vk}@valkey:6379/0') +put('cursor_secret', secrets.token_hex(32)) +put('valkey.conf', '\n'.join(['bind 0.0.0.0', 'protected-mode yes', f'requirepass {vk}', + 'dir /data', 'appendonly yes', 'appendfsync always', 'maxmemory 128mb', + 'maxmemory-policy noeviction', 'save 60 1000'])) +if not (root / '.env').exists(): + shutil.copyfile(root / '.env.example', root / '.env') +print('Local .env and secrets ready; existing values preserved.') diff --git a/scripts/linux_tests.py b/scripts/linux_tests.py new file mode 100644 index 0000000..62f3579 --- /dev/null +++ b/scripts/linux_tests.py @@ -0,0 +1,17 @@ +"""Run unchanged make targets in Linux, including the executable restart.sh.""" +import argparse +from pathlib import Path +import subprocess + +root=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser() +p.add_argument('target',choices=['check','test','test-own','test-race','test-persistence']) +args=p.parse_args() +command=['docker','run','--rm','--network','mtusi-lab2_default', + '--mount','type=bind,source='+str(root)+',target=/workspace'] +if args.target=='test-persistence': + command+=['--mount','type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock'] +command+=['mtusi-messenger:lab2-tests','make',args.target,'BASE_URL=http://api:8080'] +if args.target=='test-persistence': + command+=['ACTION_SCRIPT=scripts/restart.sh'] +raise SystemExit(subprocess.call(command,cwd=root)) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..357303c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p evidence +docker compose --env-file .env ps -q api db valkey | xargs docker inspect --format '{{.Name}} {{.State.StartedAt}}' > evidence/restart-before.txt +docker compose --env-file .env stop api +docker compose --env-file .env restart db valkey +# Do not recreate containers: this also works from a Linux test runner. +for service in db valkey; do + container=$(docker compose --env-file .env ps -q "$service") + attempts=0 + until [ "$(docker inspect --format '{{.State.Health.Status}}' "$container")" = healthy ]; do + attempts=$((attempts+1)) + [ "$attempts" -lt 40 ] || exit 1 + sleep 1 + done +done +docker compose --env-file .env start api +docker compose --env-file .env ps -q api db valkey | xargs docker inspect --format '{{.Name}} {{.State.StartedAt}}' > evidence/restart-after.txt diff --git a/store.go b/store.go new file mode 100644 index 0000000..4e0a57f --- /dev/null +++ b/store.go @@ -0,0 +1,223 @@ +package main + +import ( + "context" + "crypto/sha256" + "embed" + "encoding/hex" + "errors" + "fmt" + "sort" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrations embed.FS + +func migrate(ctx context.Context, db *pgxpool.Pool) error { + conn, err := db.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err = conn.Exec(ctx, "SELECT pg_advisory_lock(202602)"); err != nil { + return err + } + defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock(202602)") + if _, err = conn.Exec(ctx, "CREATE TABLE IF NOT EXISTS schema_migrations(version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"); err != nil { + return err + } + entries, err := migrations.ReadDir("migrations") + if err != nil { + return err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + for _, entry := range entries { + raw, err := migrations.ReadFile("migrations/" + entry.Name()) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + checksum := hex.EncodeToString(sum[:]) + var previous string + err = conn.QueryRow(ctx, "SELECT checksum FROM schema_migrations WHERE version=$1", entry.Name()).Scan(&previous) + if err == nil { + if previous != checksum { + return fmt.Errorf("migration checksum mismatch: %s", entry.Name()) + } + continue + } + if !errors.Is(err, pgx.ErrNoRows) { + return err + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err = tx.Exec(ctx, string(raw)); err == nil { + _, err = tx.Exec(ctx, "INSERT INTO schema_migrations(version,checksum) VALUES($1,$2)", entry.Name(), checksum) + } + if err != nil { + tx.Rollback(ctx) + return err + } + if err = tx.Commit(ctx); err != nil { + return err + } + } + return nil +} + +func dbError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, pgx.ErrNoRows) { + return missing() + } + var pe *pgconn.PgError + if errors.As(err, &pe) { + switch pe.Code { + case "23505": + return apiError{409, "conflict", "Username already exists"} + case "23503", "23514", "22P02", "22021": + return invalid() + } + } + return unavailable() +} +func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000000Z") } + +// pgx BytesScanner preserves every validated UTF-8 code point, including NUL. +type utf8String struct{ dst *string } + +func (s utf8String) ScanBytes(value []byte) error { + *s.dst = string(value) + return nil +} + +func scanUser(row pgx.Row) (User, error) { + var u User + var t time.Time + err := row.Scan(&u.ID, &u.Username, utf8String{&u.DisplayName}, &t) + u.CreatedAt = stamp(t) + return u, dbError(err) +} +func (a *App) user(ctx context.Context, id string) (User, error) { + return scanUser(a.db.QueryRow(ctx, "SELECT id::text,username,display_name,created_at FROM users WHERE id=$1", id)) +} + +func (a *App) createChat(ctx context.Context, title string, ids []string) (Chat, error) { + c := Chat{ID: uuid(), Title: title, MemberIDs: ids} + var t time.Time + tx, err := a.db.Begin(ctx) + if err != nil { + return c, unavailable() + } + defer tx.Rollback(ctx) + err = tx.QueryRow(ctx, "INSERT INTO chats(id,title) VALUES($1,$2) RETURNING created_at", c.ID, []byte(title)).Scan(&t) + if err != nil { + return c, dbError(err) + } + // FK failure on any participant rolls back the chat and earlier participants. + for _, id := range ids { + if _, err = tx.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", c.ID, id); err != nil { + return c, dbError(err) + } + } + if err = tx.Commit(ctx); err != nil { + return c, unavailable() + } + c.CreatedAt = stamp(t) + return c, nil +} +func (a *App) chat(ctx context.Context, id, user string) (Chat, error) { + c := Chat{} + var t time.Time + err := a.db.QueryRow(ctx, `SELECT c.id::text,c.title,c.created_at, + ARRAY(SELECT user_id::text FROM chat_members WHERE chat_id=c.id ORDER BY user_id) + FROM chats c WHERE c.id=$1 AND EXISTS(SELECT 1 FROM chat_members WHERE chat_id=c.id AND user_id=$2)`, id, user).Scan(&c.ID, utf8String{&c.Title}, &t, &c.MemberIDs) + c.CreatedAt = stamp(t) + return c, dbError(err) +} +func boundary(c cursor) (time.Time, string, error) { + if c.ID == "" { + return time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), "00000000-0000-0000-0000-000000000000", nil + } + t, err := time.Parse(time.RFC3339Nano, c.Time) + if err != nil { + return t, "", invalid() + } + id, err := normalizeID(c.ID) + return t, id, err +} +func (a *App) history(ctx context.Context, id string, limit int, c cursor, scope string) (Page[Message], error) { + result := Page[Message]{Items: []Message{}} + t, last, err := boundary(c) + if err != nil { + return result, err + } + rows, err := a.db.Query(ctx, `SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id=$1 AND (created_at,id)>($2,$3) ORDER BY messages.created_at,messages.id LIMIT $4`, id, t, last, limit+1) + if err != nil { + return result, unavailable() + } + defer rows.Close() + for rows.Next() { + var m Message + var created time.Time + if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, utf8String{&m.Text}, &created); err != nil { + return result, unavailable() + } + m.CreatedAt = stamp(created) + result.Items = append(result.Items, m) + } + if rows.Err() != nil { + return result, unavailable() + } + if len(result.Items) > limit { + result.Items = result.Items[:limit] + m := result.Items[limit-1] + next := a.encodeCursor(scope, m.CreatedAt, m.ID) + result.NextCursor = &next + } + return result, nil +} +func (a *App) listChats(ctx context.Context, user string, limit int, c cursor, scope string) (Page[Chat], error) { + result := Page[Chat]{Items: []Chat{}} + t, last, err := boundary(c) + if err != nil { + return result, err + } + rows, err := a.db.Query(ctx, `SELECT c.id::text,c.title,c.created_at, + ARRAY(SELECT user_id::text FROM chat_members WHERE chat_id=c.id ORDER BY user_id) + FROM chats c JOIN chat_members own ON own.chat_id=c.id AND own.user_id=$1 + WHERE (c.created_at,c.id)>($2,$3) ORDER BY c.created_at,c.id LIMIT $4`, user, t, last, limit+1) + if err != nil { + return result, unavailable() + } + defer rows.Close() + for rows.Next() { + var chat Chat + var created time.Time + if err = rows.Scan(&chat.ID, utf8String{&chat.Title}, &created, &chat.MemberIDs); err != nil { + return result, unavailable() + } + chat.CreatedAt = stamp(created) + result.Items = append(result.Items, chat) + } + if rows.Err() != nil { + return result, unavailable() + } + if len(result.Items) > limit { + result.Items = result.Items[:limit] + v := result.Items[limit-1] + next := a.encodeCursor(scope, v.CreatedAt, v.ID) + result.NextCursor = &next + } + return result, nil +} diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..8c96747 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,118 @@ +# Проверки lab2 вручную (PowerShell) + +Работать из корня репозитория. Сначала выполнить запуск из REPORT.md. +Проверки запускать по очереди: `infrastructure.py` и persistence перезапускают +сервисы. Они сохраняют volumes; для короткого TTL нужен свободный порт 18082. + +## API и валидация + +```powershell +$env:PYTHONUTF8 = "1" +docker compose config --quiet +docker compose ps +curl.exe -i http://127.0.0.1:8080/health/ready +make check +make test BASE_URL=http://127.0.0.1:8080 +make test-own BASE_URL=http://127.0.0.1:8080 +``` + +Ожидаются healthy, HTTP 200, PASS и два набора тестов с OK (9 + 3). +`test-own` проверяет JSON/Unicode, членство, страницы, 100 сообщений от 10 клиентов, +гонку регистрации (1×201, 9×409), пароли, X-User-Id и logout. + +Полная валидация OpenAPI, дополнительно к `make check`: + +```powershell +python -m venv .venv +.\.venv\Scripts\python.exe -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/Scripts/python.exe +``` + +## Перезапуск, отказы и индекс + +В Windows контейнер инструментов запускает исходные make targets с Linux/GCC. +Для persistence ему передаётся Docker socket: он выполняет `scripts/restart.sh`. + +```powershell +docker build -f Dockerfile.tests -t mtusi-messenger:lab2-tests . +python -X utf8 scripts/linux_tests.py test-persistence +python -X utf8 scripts/linux_tests.py test-race +python -X utf8 student-tests/infrastructure.py +python -X utf8 student-tests/explain.py +``` + +Если make не установлен, тот же runner поддерживает `check`, `test`, `test-own`. +В Linux: `chmod +x scripts/restart.sh`, затем +`make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-race`, +`make test-infra`, `make explain`. + +- Persistence: прежний токен читает прежнее сообщение после restart всех сервисов. + В `evidence/restart-before.txt` и `restart-after.txt` меняются времена их запуска. +- Infrastructure: rollback без частичного чата; две повторные миграции; остановка + PostgreSQL/Valkey → 503 не позднее 6с, anonymous → 401; восстановление данных; + TTL=2с → 401; SIGTERM и сохранение logout после рестарта. +- Race: `ok`, без предупреждений о гонках. Конкурентные HTTP-записи проверяет `test-own`. +- EXPLAIN: каждый запуск добавляет отдельный чат с 10 000 сообщениями. Ожидается + `Index Scan using messages_history_idx`, без Seq Scan/Sort, для двух страниц. + +```powershell +Get-Content evidence/infrastructure.json +Get-Content evidence/explain.txt +``` + +## Исходные файлы и ручная диагностика + +```powershell +git diff --exit-code 65003b6 -- contracts tests scripts/check_contract.py scripts/validate_openapi.py README.md COURSE.md CONTRIBUTING.md MAINTAINERS.md lab.json requirements-dev.txt +git diff 65003b6 -- Makefile +git check-ignore .env secrets/ +git ls-files .env secrets/ +$apiId = docker compose ps -q api +docker inspect --format '{{.Config.User}} {{.HostConfig.ReadonlyRootfs}} {{.State.Health.Status}}' $apiId +docker compose logs --tail 30 api +``` + +Первый diff пустой; в Makefile только дополнительные собственные targets. +`.env` и secrets игнорируются, `git ls-files` для них пустой. +Контейнер: `65532:65532 true healthy`. При ошибке смотреть код ответа и логи API. + +Сохранить вывод без отдельного скрипта: + +```powershell +python -X utf8 scripts/linux_tests.py test-own 2>&1 | Tee-Object -FilePath evidence/own.txt +$LASTEXITCODE +``` + +Код завершения должен быть 0. Токены и пароли в доказательства не копировать. +Остановка с сохранением данных: `docker compose down`; запуск: +`docker compose up -d --build --wait`. Намеренная очистка: `docker compose down -v`. + +## Оценка 5, трек 2 + +```powershell +python -X utf8 student-tests/research.py +Get-Content evidence/research.json +``` + +В Linux также `make research`. Нужны Docker и Python, основной API может быть +остановлен. Один скрипт создаёт две изолированные PostgreSQL (по 2 CPU / 512 MiB), +сравнивает индексы на 100 000 сообщений (80 000 / 15 000 / 5 000), затем выполняет +backup → новый volume → restore. Проверяются счётчики, 11 записей и SHA-256 всей +истории. Ожидается PASS; запись после backup должна отсутствовать после restore. + +Сырые замеры — `research.csv`, сводка — `research.json`, планы — `research-plans.txt`. +Dump сохраняется в `backups/` (исключён из Git). По умолчанию временные контейнеры +и volumes удаляются; основной стенд сохраняется. Для ручного осмотра: + +```powershell +python -X utf8 student-tests/research.py --keep +$r = Get-Content evidence/research.json -Raw | ConvertFrom-Json +docker exec $r.restore.target_container psql -U messenger -d messenger -c "SELECT chat_id,count(*) FROM messages GROUP BY chat_id;" +``` + +После осмотра удалить только временные ресурсы этого прогона: + +```powershell +docker rm -f $r.restore.source_container $r.restore.target_container +docker volume rm $r.restore.source_volume $r.restore.new_volume +``` diff --git a/student-tests/acceptance.py b/student-tests/acceptance.py new file mode 100644 index 0000000..d44f53b --- /dev/null +++ b/student-tests/acceptance.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Independent black-box tests: python student-tests/acceptance.py [--base-url URL].""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sys +import unittest +import uuid +from urllib.parse import quote + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +BASE = 'http://127.0.0.1:8080' + + +class Acceptance(unittest.TestCase): + def setUp(self): + self.c = Client(BASE) + self.alice, self.bob, self.eve, self.chat = self.c.bootstrap() + self.path = '/api/v1/chats/' + self.chat['id'] + + def error(self, method, path, status, code, **kwargs): + result = self.c.call(method, path, status, **kwargs) + self.assertEqual(result['json']['error']['code'], code) + + def test_invalid_json(self): + for body in (b'{', b'{}{}', b'null', b'[]', b'{"text":null}', + b'{"text":1}', b'{"Text":"x"}', b'{"text":"x","extra":0}', + b'{"text":"x","text":"y"}', b'{"text":"\xff"}', + b'{"text":"\\ud800"}', b'{"text":"\\udc00"}'): + with self.subTest(body=body): + self.error('POST', self.path+'/messages', 400, 'invalid_request', + actor=self.alice, raw=body, headers={'Content-Type': 'application/json'}) + self.error('POST', self.path+'/messages', 415, 'unsupported_media_type', + actor=self.alice, raw=b'{"text":"x"}', headers={'Content-Type':'text/plain'}) + + def test_foreign_chat_and_authentication(self): + for path in (self.path, self.path+'/messages'): + self.error('GET', path, 404, 'not_found', actor=self.eve) + self.error('GET', path, 401, 'unauthorized') + self.error('POST', self.path+'/messages', 404, 'not_found', actor=self.eve, raw=b'{') + self.error('GET', '/api/v1/users/me', 401, 'unauthorized', headers={'X-User-Id':self.alice['id']}) + self.error('GET', '/api/v1/users/me', 401, 'unauthorized', headers={'X-User-Id':'broken'}) + for actor in (self.alice, self.bob): + me = self.c.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'] + self.assertEqual(me, actor['user']) + msg = self.c.message(actor, self.chat, 'identity')['json'] + self.assertEqual(msg['sender_id'], actor['id']) + self.error('POST', self.path+'/messages', 400, 'invalid_request', actor=self.alice, + body={'text':'forged','sender_id':self.bob['id']}) + + def test_empty_and_unicode_message(self): + for text in ('', ' \t\n', '\u2003\u00a0', '😀'*4001): + self.error('POST', self.path+'/messages', 400, 'invalid_request', actor=self.alice, body={'text':text}) + for text in ('Привет, 世界 👋 e\u0301', '😀'*4000, '\ufffd', 'literal \\ud800', '\u0000'*4000): + sent = self.c.message(self.alice, self.chat, text)['json'] + self.assertEqual(sent['text'], text) + escaped = self.c.call('POST', self.path+'/messages', 201, actor=self.alice, + raw=b'{"text":"\\ud83d\\ude00"}', + headers={'Content-Type':'application/json'}, schema='Message')['json'] + self.assertEqual(escaped['text'], '😀') + history = self.c.call('GET', self.path+'/messages', actor=self.bob, schema='MessagePage')['json'] + self.assertEqual(len(history['items']), 6) + + def test_duplicate_username_and_user_validation(self): + self.error('POST', '/api/v1/users', 409, 'conflict', body=self.alice['registration']) + for name in ('ab', 'Upper', 'русский', 'a'*33, 'a b'): + self.error('POST', '/api/v1/users', 400, 'invalid_request', body={'username':name,'display_name':'X','password':'test-password-123'}) + for display in (' ', '😀'*101, None, 1): + self.error('POST', '/api/v1/users', 400, 'invalid_request', body={'username':'valid_name','display_name':display,'password':'test-password-123'}) + self.c.call('POST', '/api/v1/users', 201, body={'username':'u_'+uuid.uuid4().hex[:20], 'display_name':'😀'*100,'password':'test-password-123'}, schema='User') + + def test_membership_validation(self): + for ids in ([], [self.bob['id']], [self.alice['id']]*2, [str(uuid.uuid4()),self.alice['id']], + [self.alice['id'],'bad'], None, [None], [self.alice['id'].upper(), self.alice['id']]): + self.error('POST', '/api/v1/chats', 400, 'invalid_request', actor=self.alice, body={'title':'chat','member_ids':ids}) + for title in (' ', '😀'*101, None): + self.error('POST', '/api/v1/chats', 400, 'invalid_request', actor=self.alice, body={'title':title,'member_ids':[self.alice['id']]}) + listed = self.c.call('GET', '/api/v1/chats', actor=self.alice, schema='ChatPage')['json'] + self.assertEqual(len(listed['items']), 1) + + def pages(self, path, actor, schema, sizes=(3, 7, 2)): + items, cursor, page_number = [], None, 0 + while True: + query = '?limit='+str(sizes[page_number % len(sizes)]) + if cursor: + query += '&cursor='+quote(cursor, safe='') + result = self.c.call('GET', path+query, actor=actor, schema=schema)['json'] + items.extend(result['items']) + cursor = result['next_cursor'] + page_number += 1 + self.assertLess(page_number, 1000) + if cursor is None: + break + self.assertEqual(len(items), len({item['id'] for item in items})) + self.assertEqual(items, sorted(items, key=lambda item:(item['created_at'], item['id']))) + return items + + def test_multiple_pages_and_cursor_scope(self): + sent = [self.c.message(self.alice, self.chat, 'page '+str(i))['json'] for i in range(23)] + self.assertEqual(self.pages(self.path+'/messages', self.bob, 'MessagePage'), sent) + for _ in range(11): + self.c.call('POST', '/api/v1/chats', 201, actor=self.alice, + body={'title':'page','member_ids':[self.alice['id']]}, schema='Chat') + self.assertEqual(len(self.pages('/api/v1/chats',self.alice,'ChatPage')),12) + self.assertEqual(len(self.pages('/api/v1/chats',self.bob,'ChatPage')),1) + cursor = self.c.call('GET', self.path+'/messages?limit=1',actor=self.alice,schema='MessagePage')['json']['next_cursor'] + other = self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json'] + self.error('GET','/api/v1/chats/'+other['id']+'/messages?cursor='+quote(cursor),400,'invalid_request',actor=self.alice) + for query in ('limit=0','limit=101','limit=1.5','limit=true','limit=','limit=1&limit=2','cursor=', 'cursor=bad', 'cursor='+quote(cursor[:-3]+'xxx')): + self.error('GET',self.path+'/messages?'+query,400,'invalid_request',actor=self.alice) + first = self.c.call('GET', self.path+'/messages?limit=23', actor=self.alice, schema='MessagePage')['json'] + self.assertIsNone(first['next_cursor']) + + def test_empty_history_default_limit_and_concurrent_append(self): + empty = self.c.call('GET', self.path+'/messages', actor=self.alice, schema='MessagePage')['json'] + self.assertEqual(empty, {'items':[], 'next_cursor':None}) + sent = [self.c.message(self.alice,self.chat,'initial '+str(i))['json'] for i in range(51)] + first = self.c.call('GET', self.path+'/messages', actor=self.bob, schema='MessagePage')['json'] + self.assertEqual(first['items'], sent[:50]) + self.assertIsNotNone(first['next_cursor']) + appended = self.c.message(self.bob,self.chat,'after page one')['json'] + second = self.c.call('GET', self.path+'/messages?limit=100&cursor='+quote(first['next_cursor'],safe=''), + actor=self.alice, schema='MessagePage')['json'] + self.assertEqual(second['items'], [sent[-1], appended]) + self.assertIsNone(second['next_cursor']) + + def test_chat_cursor_bound_to_current_user(self): + self.c.call('POST','/api/v1/chats',201,actor=self.alice, + body={'title':'second shared chat','member_ids':[self.alice['id'],self.bob['id']]},schema='Chat') + first = self.c.call('GET','/api/v1/chats?limit=1',actor=self.alice,schema='ChatPage')['json'] + cursor = quote(first['next_cursor'],safe='') + self.error('GET','/api/v1/chats?cursor='+cursor,400,'invalid_request',actor=self.bob) + last = self.c.call('GET','/api/v1/chats?limit=100&cursor='+cursor,actor=self.alice,schema='ChatPage')['json'] + self.assertEqual(len(last['items']),1) + self.assertIsNone(last['next_cursor']) + + def test_100_messages_from_10_clients(self): + actors = [self.c.user('worker') for _ in range(10)] + chat = self.c.call('POST','/api/v1/chats',201,actor=actors[0], + body={'title':'concurrency','member_ids':[a['id'] for a in actors]},schema='Chat')['json'] + def worker(index): + client = Client(BASE) + return [client.message(actors[index],chat,'client %d message %d' % (index,j))['json'] for j in range(10)] + with ThreadPoolExecutor(max_workers=10) as pool: + sent = [message for batch in pool.map(worker, range(10)) for message in batch] + read = self.pages('/api/v1/chats/'+chat['id']+'/messages',actors[0],'MessagePage',(17,)) + self.assertEqual(len(read),100) + self.assertEqual({m['id'] for m in read},{m['id'] for m in sent}) + self.assertEqual({m['text'] for m in read},{m['text'] for m in sent}) + print('CONCURRENCY: 10 clients x 10 writes = 100 acknowledged, 100 read, 100 unique',flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default=BASE) + args, remaining = parser.parse_known_args() + BASE = args.base_url + unittest.main(argv=[sys.argv[0]]+remaining, verbosity=2) diff --git a/student-tests/authentication.py b/student-tests/authentication.py new file mode 100644 index 0000000..66caa02 --- /dev/null +++ b/student-tests/authentication.py @@ -0,0 +1,70 @@ +"""Session, username race, Unicode password and identity boundary tests.""" +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import unittest +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +BASE = 'http://127.0.0.1:8080' + + +def basic(username, password): + return {'Authorization': 'Basic ' + base64.b64encode((username+':'+password).encode()).decode()} + + +class Authentication(unittest.TestCase): + def setUp(self): + self.c = Client(BASE) + self.a = self.c.user('auth') + + def test_wrong_password_logout_and_identity(self): + reg = self.a['registration'] + self.c.call('POST', '/api/v1/auth/sessions', 401, headers=basic(reg['username'], 'wrong-password')) + self.c.call('POST', '/api/v1/auth/sessions', 401, headers=basic('missing_'+uuid.uuid4().hex[:12], 'wrong-password')) + self.c.call('GET', '/api/v1/users/me', 401, headers=basic(reg['username'], reg['password'])) + self.c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': self.a['id']}) + me = self.c.call('GET', '/api/v1/users/me', actor=self.a, headers={'X-User-Id':str(uuid.uuid4())}, schema='User')['json'] + self.assertEqual(me['id'], self.a['id']) + self.c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.a) + self.c.call('GET', '/api/v1/users/me', 401, actor=self.a) + self.c.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=self.a) + self.c.call('POST', '/api/v1/auth/sessions', 201, headers=basic(reg['username'],reg['password']), schema='Session') + + def test_password_boundaries_unicode_and_colon(self): + name = 'pass_'+uuid.uuid4().hex[:16] + for password in ('short', '😀'*129, None, 123): + self.c.call('POST', '/api/v1/users', 400, body={'username':name,'display_name':'пароль','password':password}) + password = '😀'*127 + ':' + user = self.c.call('POST', '/api/v1/users', 201, body={'username':name,'display_name':'\u0000'*100,'password':password}, schema='User')['json'] + self.assertEqual(user['display_name'],'\u0000'*100) + session = self.c.call('POST', '/api/v1/auth/sessions', 201, headers=basic(name,password), schema='Session')['json'] + self.assertEqual(session['user'],user) + actor = {'token':session['token']} + chat = self.c.call('POST','/api/v1/chats',201,actor=actor,body={'title':'\u0000'*100,'member_ids':[user['id']]},schema='Chat')['json'] + read = self.c.call('GET','/api/v1/chats/'+chat['id'],actor=actor,schema='Chat')['json'] + self.assertEqual(read['title'],'\u0000'*100) + self.c.call('POST', '/api/v1/auth/sessions', 401, headers=basic(name,'😀'*127+'!')) + self.c.call('POST', '/api/v1/auth/sessions', 400, headers=basic(name,password), body={}) + + def test_registration_race(self): + body = {'username':'race_'+uuid.uuid4().hex[:16], 'display_name':'race', 'password':'race-password-123'} + def register(_): + return Client(BASE).call('POST','/api/v1/users', [201,409], body=body)['status'] + with ThreadPoolExecutor(max_workers=10) as pool: + statuses = list(pool.map(register,range(10))) + self.assertEqual(statuses.count(201),1) + self.assertEqual(statuses.count(409),9) + print('REGISTRATION RACE: 10 requests, 1 created, 9 conflicts',flush=True) + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--base-url',default=BASE) + args,remaining = p.parse_known_args() + BASE = args.base_url + unittest.main(argv=[sys.argv[0]]+remaining,verbosity=2) diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 0000000..e585700 --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,40 @@ +"""Create a dedicated 10,000-message chat; record the real history query plans.""" +from pathlib import Path +import sys +import uuid + +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client, check +from infrastructure import sql, ROOT + +c=Client('http://127.0.0.1:8080') +a=c.user('explain') +chat=c.call('POST','/api/v1/chats',201,actor=a,body={'title':'EXPLAIN 10000','member_ids':[a['id']]},schema='Chat')['json'] +# IDs below are generated by the API, not external SQL input. Data is synthetic. +cid=str(uuid.UUID(chat['id'])) +uid=str(uuid.UUID(a['id'])) +sql(f"""INSERT INTO messages(id,chat_id,sender_id,text,created_at) + SELECT gen_random_uuid(),'{cid}','{uid}',convert_to('synthetic history row '||g,'UTF8'), + '2026-01-01T00:00:00Z'::timestamptz+g*interval '1 millisecond' + FROM generate_series(1,10000) AS g; + ANALYZE messages;""") +check(sql(f"SELECT count(*) FROM messages WHERE chat_id='{cid}';")=='10000','seed row count') +plans=[] +for label,bound in [('first','2026-01-01T00:00:00Z'),('deep (after 9000 rows)','2026-01-01T00:00:09Z')]: + last='00000000-0000-0000-0000-000000000000' + if label.startswith('deep'): + last=str(uuid.UUID(sql(f"SELECT id FROM messages WHERE chat_id='{cid}' AND created_at='{bound}'::timestamptz;"))) + query=f"""EXPLAIN (ANALYZE, BUFFERS) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='{cid}' AND (created_at,id)>('{bound}'::timestamptz,'{last}'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51;""" + plan=sql(query) + if 'messages_history_idx' not in plan or 'Seq Scan' in plan or 'Sort ' in plan: + print(plan) + raise AssertionError('history query must use ordered index') + plans.append(label+'\n'+query+'\n'+plan) +path=ROOT/'evidence/explain.txt' +path.parent.mkdir(exist_ok=True) +path.write_text('PostgreSQL: '+sql('SELECT version();')+'\nRows in dedicated chat: 10000\n\n'+'\n\n'.join(plans)+'\n',encoding='utf-8') +print('PASS: 10000 messages; first and deep page use messages_history_idx without Seq Scan/Sort') +print(path.relative_to(ROOT)) diff --git a/student-tests/infrastructure.py b/student-tests/infrastructure.py new file mode 100644 index 0000000..12144d1 --- /dev/null +++ b/student-tests/infrastructure.py @@ -0,0 +1,134 @@ +"""Destructive only to this Compose lab's running processes; volumes preserved. +Run on an idle lab stand: python student-tests/infrastructure.py. +""" +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time +import urllib.error +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check + + +def docker(*args, data=None): + p = subprocess.run(['docker', *args], cwd=ROOT, input=data, text=True, + encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=90) + if p.returncode: + raise Failure('docker '+args[0]+' failed: '+p.stderr[-1000:]) + return p.stdout.strip() + + +def compose(*args, data=None): + return docker('compose', *args, data=data) + + +def sql(query): + return compose('exec','-T','db','psql','-X','-v','ON_ERROR_STOP=1','-U','messenger','-d','messenger','-At',data=query) + + +def ready(c): + start = time.monotonic() + while time.monotonic()-start < 45: + try: + c.call('GET','/health/ready') + return round(time.monotonic()-start,3) + except Failure: + time.sleep(.2) + raise Failure('readiness deadline exceeded') + + +def counts(): + return sql('SELECT (SELECT count(*) FROM users),(SELECT count(*) FROM chats),(SELECT count(*) FROM chat_members),(SELECT count(*) FROM messages),(SELECT count(*) FROM schema_migrations);') + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--base-url',default='http://127.0.0.1:8080') + args = p.parse_args() + c = Client(args.base_url,timeout=8) + ready(c) + a,b,e,chat = c.bootstrap() + sent = c.message(a,chat,'durability probe')['json'] + out = {'utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'outages':[]} + before = counts() + c.call('POST','/api/v1/chats',400,actor=a,body={'title':'rollback probe','member_ids':[a['id'],b['id'],str(uuid.uuid4())]}) + after = counts() + check(before==after,'partial chat survived rollback') + out['rollback']={'before':before,'after':after,'result':'PASS: no chat or membership left'} + print('ROLLBACK: table counts unchanged after FK failure',flush=True) + for _ in range(2): + compose('run','--rm','--no-deps','api','migrate') + check(counts()==before,'migrations changed existing data') + out['migrations']='PASS: two repeated migrations, counts unchanged' + print('MIGRATIONS: repeat twice, counts unchanged',flush=True) + for service in ('db','valkey'): + container = compose('ps','-q',service) + old_start = docker('inspect','--format','{{.State.StartedAt}}',container) + compose('stop',service) + try: + timings=[] + for method,path,body in [('GET','/api/v1/users/me',None),('POST','/api/v1/chats/'+chat['id']+'/messages',{'text':'must not commit'})]: + response=c.call(method,path,503,actor=a,body=body) + check(response['elapsed'] < 6,'dependency error exceeded time budget') + timings.append(round(response['elapsed'],3)) + c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':a['id']}) + c.call('GET','/health/live',200) + c.call('GET','/health/ready',503) + finally: + compose('start',service) + recovery=ready(c) + new_start=docker('inspect','--format','{{.State.StartedAt}}',container) + check(old_start!=new_start,'dependency did not restart') + read=c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=a,schema='MessagePage')['json']['items'] + check(read==[sent],'data missing or rejected write committed') + out['outages'].append({'service':service,'statuses':[503,503,401,200,503],'request_seconds':timings,'ready_seconds_after_start':recovery,'started_before':old_start,'started_after':new_start,'same_token_and_message':True}) + print(service.upper()+': bounded 503, anonymous 401, recovery with same token/message',flush=True) + name='mtusi-lab2-expiry-test' + check(not docker('ps','-aq','--filter','name=^/'+name+'$'),'expiry test container name already occupied') + try: + compose('run','-d','--no-deps','--name',name,'-p','127.0.0.1:18082:8080','-e','SESSION_TTL_SECONDS=2','api') + short=Client('http://127.0.0.1:18082',timeout=8) + ready(short) + actor=short.user('expiry') + short.call('GET','/api/v1/users/me',actor=actor,schema='User') + time.sleep(2.2) + short.call('GET','/api/v1/users/me',401,actor=actor) + out['expiry']='PASS: separate API TTL=2s, 200 then 401 after 2.2s' + print('EXPIRY: TTL=2s, 200 then 401',flush=True) + finally: + docker('rm','-f',name) + start=time.monotonic() + compose('stop','-t','10','api') + elapsed=time.monotonic()-start + api=compose('ps','-aq','api') + state=json.loads(docker('inspect','--format','{{json .State}}',api)) + try: + check(elapsed<10 and state['ExitCode']==0 and not state['OOMKilled'],'unclean shutdown') + try: + c.call('GET','/health/live') + except Failure: + pass + else: + raise Failure('API still accepts requests after stop') + finally: + compose('start','api') + ready(c) + c.call('GET','/api/v1/users/me',actor=a,schema='User') + out['sigterm']={'seconds':round(elapsed,3),'exit_code':state['ExitCode'],'oom_killed':state['OOMKilled'],'no_new_requests':True} + c.call('DELETE','/api/v1/auth/sessions/current',204,actor=a) + compose('restart','valkey') + ready(c) + c.call('GET','/api/v1/users/me',401,actor=a) + out['logout_restart']='PASS: revoked session stays revoked after Valkey restart' + (ROOT/'evidence').mkdir(exist_ok=True) + (ROOT/'evidence/infrastructure.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n',encoding='utf-8') + print('PASS: infrastructure, expiry, rollback, migrations, shutdown and durable logout',flush=True) + + +if __name__=='__main__': + main() diff --git a/student-tests/research.py b/student-tests/research.py new file mode 100644 index 0000000..3327404 --- /dev/null +++ b/student-tests/research.py @@ -0,0 +1,227 @@ +"""Grade 5, track 2: isolated index experiment and verified PostgreSQL restore. + +Only synthetic data, standard library + Docker. Main Compose volumes are untouched. +""" +import argparse +import csv +import datetime as dt +import hashlib +import json +import math +from pathlib import Path +import statistics +import subprocess +import time +import uuid + +from infrastructure import ROOT, docker + +IMAGE = 'postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3' + + +def key(kind, number): + # Deterministic fixture UUIDs only; password hashing remains Argon2id in auth.go. + return str(uuid.UUID(hashlib.md5(f'lab2-{kind}-{number}'.encode()).hexdigest())) + + +def sql(container, query): + return docker('exec', '-i', container, 'psql', '-XqAt', '-v', 'ON_ERROR_STOP=1', + '-U', 'messenger', '-d', 'messenger', data=query) + + +def plans(container, query): + raw = sql(container, query).strip() + result, decoder = [], json.JSONDecoder() + while raw: + value, end = decoder.raw_decode(raw) + result.append(value[0]) + raw = raw[end:].lstrip() + return result + + +def start(name): + docker('run', '-d', '--name', name, '--network', 'none', '--memory', '512m', '--cpus', '2', + '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', '-e', 'POSTGRES_USER=messenger', + '-e', 'POSTGRES_DB=messenger', '--mount', f'type=volume,source={name}-data,target=/var/lib/postgresql/data', + IMAGE, '-c', 'max_parallel_workers_per_gather=0', '-c', 'jit=off') + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + try: + # initdb temporarily starts a socket-only server: require final PID 1. + docker('exec', name, 'pg_isready', '-h', '127.0.0.1', '-U', 'messenger') + return + except AssertionError: + time.sleep(.2) + raise RuntimeError('PostgreSQL startup deadline exceeded') + + +def fingerprint(container): + query = """COPY (SELECT id,chat_id,sender_id,encode(text,'hex'), + to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS.US') + FROM messages ORDER BY id) TO STDOUT;""" + command = ['docker', 'exec', container, 'psql', '-XqAt', '-v', 'ON_ERROR_STOP=1', + '-U', 'messenger', '-d', 'messenger', '-c', query] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + digest = hashlib.sha256() + for chunk in iter(lambda: process.stdout.read(1024*1024), b''): + digest.update(chunk) + error = process.stderr.read().decode('utf-8', errors='replace') + if process.wait() != 0: + raise RuntimeError('Fingerprint failed: '+error[-300:]) + return digest.hexdigest() + + +def snapshot(container): + counts = {table: int(sql(container, f'SELECT count(*) FROM {table};')) + for table in ('users', 'chats', 'chat_members', 'messages', 'schema_migrations')} + ids = ','.join("'"+key('message', n)+"'" for n in (1, 2, 40000, 79999, 80000, 80001, 90000, 95000, 95001, 99999, 100000)) + sample = json.loads(sql(container, f"""SELECT json_agg(t ORDER BY id) FROM ( + SELECT id,chat_id,sender_id,encode(text,'hex') AS utf8_hex,created_at FROM messages + WHERE id IN ({ids})) t;""")) + assert len(sample) == 11 + chat_counts = json.loads(sql(container,"SELECT json_object_agg(chat_id,n) FROM (SELECT chat_id,count(*) n FROM messages GROUP BY chat_id) t;")) + assert sorted(chat_counts.values()) == [5000,15000,80000] + return {'counts': counts, 'chat_counts':chat_counts, 'history_sha256': fingerprint(container), + 'samples': {row['id']: hashlib.sha256(json.dumps(row,sort_keys=True).encode()).hexdigest() for row in sample}} + + +def percentile(values, fraction): + return sorted(values)[math.ceil(len(values)*fraction)-1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--keep', action='store_true', help='Keep the two temporary containers/volumes for inspection') + args = parser.parse_args() + suffix = uuid.uuid4().hex[:10] + source, target = 'lab2-research-'+suffix, 'lab2-restore-'+suffix + created, raw, saved_plans = [], [], [] + user, chats = key('user', 1), [key('chat', n) for n in (1, 2, 3)] + evidence = ROOT/'evidence' + evidence.mkdir(exist_ok=True) + out = {'utc':dt.datetime.now(dt.timezone.utc).isoformat(), 'image':IMAGE, + 'host':docker('info','--format','{{.OSType}}/{{.Architecture}} CPUs={{.NCPU}} MemoryBytes={{.MemTotal}}'), + 'limits':{'cpus_per_container':2,'memory_bytes':512*1024*1024,'clients':1}, + 'method':{'repeats':3,'read_samples_per_case_per_repeat':20,'warmup':3, + 'order':['chat,composite','composite,chat','chat,composite'], + 'distribution':[80000,15000,5000],'parallel_query':False,'jit':False}} + insert = f"""INSERT INTO messages(id,chat_id,sender_id,text,created_at) + SELECT md5('lab2-message-'||g)::uuid, + CASE WHEN g<=80000 THEN '{chats[0]}'::uuid WHEN g<=95000 THEN '{chats[1]}'::uuid ELSE '{chats[2]}'::uuid END, + '{user}'::uuid,convert_to('Синтетическое сообщение №'||g||repeat(' x',40),'UTF8') + ||CASE WHEN g%10000=0 THEN decode('00','hex') ELSE ''::bytea END, + '2026-01-01T00:00:00Z'::timestamptz+g*interval '1 millisecond' + FROM generate_series(1,100000) g;""" + try: + created.append(source) + start(source) + out['postgres'] = sql(source,'SELECT version();') + sql(source,'CREATE TABLE schema_migrations(version text PRIMARY KEY,checksum text NOT NULL,applied_at timestamptz NOT NULL DEFAULT clock_timestamp());') + for migration in sorted((ROOT/'migrations').glob('*.sql')): + content = migration.read_bytes() + checksum = hashlib.sha256(content).hexdigest() + sql(source, "BEGIN;\n"+content.decode('utf-8')+f"\nINSERT INTO schema_migrations(version,checksum) VALUES('{migration.name}','{checksum}'); COMMIT;") + # Valid PHC shape, no known password: this fixture user is never used for login. + fixture_hash = '$argon2id$v=19$m=65536,t=3,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + sql(source,f"INSERT INTO users(id,username,display_name,password_hash) VALUES('{user}','research_user',convert_to('Synthetic fixture','UTF8'),'{fixture_hash}');") + for index, chat in enumerate(chats): + sql(source,f"INSERT INTO chats(id,title) VALUES('{chat}',convert_to('research {index}','UTF8')); INSERT INTO chat_members VALUES('{chat}','{user}');") + cases = [('hot_first',chats[0],0),('hot_deep',chats[0],70000),('small_deep',chats[2],99000)] + for repeat in range(3): + for variant in (('chat','composite') if repeat%2==0 else ('composite','chat')): + index_name = 'research_chat_idx' if variant=='chat' else 'messages_history_idx' + columns = 'chat_id' if variant=='chat' else 'chat_id,created_at,id' + sql(source,f'TRUNCATE messages; DROP INDEX IF EXISTS messages_history_idx; DROP INDEX IF EXISTS research_chat_idx; CREATE INDEX {index_name} ON messages({columns});') + before = time.monotonic() + write = plans(source,'EXPLAIN (ANALYZE, BUFFERS, WAL, FORMAT JSON) '+insert)[0] + wall = (time.monotonic()-before)*1000 + assert sql(source,'SELECT count(*) FROM messages;')=='100000' + sql(source,'ANALYZE messages;') + size = int(sql(source,f"SELECT pg_relation_size('{index_name}');")) + raw.append({'repeat':repeat+1,'variant':variant,'case':'insert_100000', + 'execution_ms':write['Execution Time'],'wall_with_commit_and_docker_ms':wall, + 'wal_bytes':write['Plan'].get('WAL Bytes',0),'index_bytes':size}) + if repeat==0: + saved_plans.append(variant+'/insert_100000 (EXPLAIN JSON)\n'+json.dumps(write,ensure_ascii=False)) + for case, chat, number in cases: + boundary = key('message',number) if number else '00000000-0000-0000-0000-000000000000' + query = f"""EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id::text,chat_id::text,sender_id::text,text,created_at FROM messages + WHERE chat_id='{chat}' AND (created_at,id)>( + '2026-01-01T00:00:00Z'::timestamptz+{number}*interval '1 millisecond','{boundary}'::uuid) + ORDER BY messages.created_at,messages.id LIMIT 51;""" + plans(source,query*3) + measured = plans(source,query*20) + for plan in measured: + assert plan['Plan']['Actual Rows']==51 + raw.append({'repeat':repeat+1,'variant':variant,'case':case, + 'execution_ms':plan['Execution Time'],'planning_ms':plan['Planning Time'], + 'shared_hit_blocks':plan['Plan'].get('Shared Hit Blocks',0), + 'shared_read_blocks':plan['Plan'].get('Shared Read Blocks',0)}) + if repeat==0: + saved_plans.append(variant+'/'+case+'\n'+query+'\n'+sql(source,query.replace(', FORMAT JSON',''))) + print(f'RESEARCH: repeat {repeat+1}/3, {variant}, 100000 rows, insert {write["Execution Time"]:.1f} ms',flush=True) + summary=[] + for variant in ('chat','composite'): + for case in ('insert_100000','hot_first','hot_deep','small_deep'): + samples=[r for r in raw if r['variant']==variant and r['case']==case] + values=[r['execution_ms'] for r in samples] + row={'variant':variant,'case':case,'samples':len(values),'median_ms':statistics.median(values), + 'p95_ms':percentile(values,.95) if len(values)>3 else None, + 'p99_ms':percentile(values,.99) if len(values)>3 else None} + for metric in ('index_bytes','wal_bytes','shared_hit_blocks','shared_read_blocks'): + row[metric]=statistics.median([s[metric] for s in samples]) if metric in samples[0] else None + summary.append(row) + out['summary']=summary + expected = snapshot(source) + assert expected['counts']['messages']==100000 + backup_dir = ROOT/'backups' + backup_dir.mkdir(exist_ok=True) + archive = backup_dir/f'research-{suffix}.dump' + before = time.monotonic() + docker('exec',source,'pg_dump','-U','messenger','-d','messenger','-Fc','-f','/tmp/research.dump') + dump_seconds = time.monotonic()-before + docker('cp',source+':/tmp/research.dump',str(archive)) + marker = key('post-backup',1) + sql(source,f"INSERT INTO messages VALUES('{marker}','{chats[0]}','{user}',convert_to('Committed after backup','UTF8'),clock_timestamp());") + assert sql(source,'SELECT count(*) FROM messages;')=='100001' + recovery_start = time.monotonic() + created.append(target) + start(target) + startup_seconds = time.monotonic()-recovery_start + assert sql(target,"SELECT count(*) FROM pg_tables WHERE schemaname='public';")=='0' + docker('cp',str(archive),target+':/tmp/research.dump') + before = time.monotonic() + docker('exec',target,'pg_restore','--exit-on-error','-U','messenger','-d','messenger','/tmp/research.dump') + restore_seconds = time.monotonic()-before + actual = snapshot(target) + assert actual==expected, 'Restored counts, selected messages or complete history differ' + assert sql(target,f"SELECT count(*) FROM messages WHERE id='{marker}';")=='0' + out['restore']={'source_container':source,'target_container':target, + 'source_volume':source+'-data','new_volume':target+'-data', + 'destination_initial_public_tables':0,'dump_seconds':dump_seconds, + 'dump_bytes':archive.stat().st_size,'dump_path':archive.relative_to(ROOT).as_posix(), + 'destination_startup_seconds':startup_seconds,'pg_restore_seconds':restore_seconds, + 'recovery_through_verification_seconds':time.monotonic()-recovery_start, + 'before_backup':expected,'after_restore':actual, + 'source_count_after_backup':100001,'restored_count':100000,'post_backup_marker_absent':True} + out['result']='PASS: two indexes, 100000 uneven messages, write cost, new-volume restore and SHA256 verified' + (evidence/'research.json').write_text(json.dumps(out,ensure_ascii=False,indent=2)+'\n',encoding='utf-8') + (evidence/'research-plans.txt').write_text('\n\n'.join(saved_plans)+'\n',encoding='utf-8') + with (evidence/'research.csv').open('w',encoding='utf-8',newline='') as file: + writer=csv.DictWriter(file,fieldnames=list(dict.fromkeys(k for row in raw for k in row))) + writer.writeheader() + writer.writerows(raw) + print(out['result'],flush=True) + print(f'RESTORE: {restore_seconds:.3f}s; same 100000 rows, 11 samples and full SHA256; later COMMIT absent',flush=True) + if args.keep: + print('KEPT: '+source+' '+target,flush=True) + finally: + if not args.keep: + for name in reversed(created): + subprocess.run(['docker','rm','-f',name],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + subprocess.run(['docker','volume','rm',name+'-data'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + + +if __name__=='__main__': + main()