From da5caa9137bfdd8c5807f14ce4a02c00f9a71563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B8=D1=86=20=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8?= =?UTF-8?q?=D0=B9?= Date: Fri, 6 Nov 2026 01:53:02 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=202=20=E2=80=94=20PostgreSQL=20?= =?UTF-8?q?=D0=B8=20=D1=81=D0=B5=D1=81=D1=81=D0=B8=D0=B8=20Redis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 12 +- .gitattributes | 1 + Dockerfile | 31 ++ REPORT.md | 132 +++++-- compose.yaml | 59 ++++ evidence/compose-checks.txt | 43 +++ evidence/history-plan.txt | 38 ++ requirements.txt | 3 + scripts/explain-history.sql | 33 ++ scripts/restart.cmd | 2 + scripts/restart.sh | 4 + src/healthcheck.py | 12 + src/messenger/__init__.py | 3 + src/messenger/__main__.py | 58 +++ src/messenger/config.py | 68 ++++ src/messenger/errors.py | 68 ++++ src/messenger/http_api.py | 432 +++++++++++++++++++++++ src/messenger/limits.py | 166 +++++++++ src/messenger/logs.py | 27 ++ src/messenger/migrate.py | 31 ++ src/messenger/migrations/001_initial.sql | 34 ++ src/messenger/pagination.py | 45 +++ src/messenger/sessions.py | 96 +++++ src/messenger/store.py | 140 ++++++++ src/messenger/validation.py | 129 +++++++ student-tests/grade4.py | 141 ++++++++ 26 files changed, 1775 insertions(+), 33 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/compose-checks.txt create mode 100644 evidence/history-plan.txt create mode 100644 requirements.txt create mode 100644 scripts/explain-history.sql create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 src/healthcheck.py create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/__main__.py create mode 100644 src/messenger/config.py create mode 100644 src/messenger/errors.py create mode 100644 src/messenger/http_api.py create mode 100644 src/messenger/limits.py create mode 100644 src/messenger/logs.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/migrations/001_initial.sql create mode 100644 src/messenger/pagination.py create mode 100644 src/messenger/sessions.py create mode 100644 src/messenger/store.py create mode 100644 src/messenger/validation.py create mode 100644 student-tests/grade4.py diff --git a/.env.example b/.env.example index 714c8076..85c3e0f1 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,5 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Local stand. Replace the empty values with independently generated secrets. APP_PORT=8080 -APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +POSTGRES_PASSWORD= +CURSOR_SECRET= diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..e9b813f5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +scripts/restart.sh text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..99f81fa3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +# Unprivileged user with a fixed UID; the application never runs as root. +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +WORKDIR /app +COPY requirements.txt /app/ +RUN pip install --no-cache-dir -r requirements.txt + +# Only the source code goes into the image (no .env, tests, evidence, .git). +# Files stay owned by root, so the app user cannot modify its own code. +COPY src/ /app/src/ + +USER 10001:10001 +EXPOSE 8080 + +# docker stop sends SIGTERM; the app drains and exits well within the 10 s budget. +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "/app/src/healthcheck.py"] + +# Exec form: python is PID 1 and receives signals directly (no shell in between). +CMD ["python", "-m", "messenger"] diff --git a/REPORT.md b/REPORT.md index e42f2b05..82a7ee75 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,121 @@ -# Отчёт — лабораторная 2 +# Лабораторная 2. Мессенджер с постоянным хранилищем -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Риц Дмитрий, БВТ2304** -## Запуск из чистого clone +Целевая оценка — **4**. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +## Запуск на Windows -## Архитектура и решения +Для запуска нужны Docker Desktop с Linux Engine и Docker Compose. Тесты запускаются на хосте через Python 3.9+. В контейнерах используются Python 3.12.12, PostgreSQL 17.6 и Redis 7.4.6. Команды ниже выполняются из корня репозитория. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```powershell +if (-not (Test-Path .env)) { + python -c 'from pathlib import Path; import secrets; Path(".env").write_text(f"APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nPOSTGRES_PASSWORD={secrets.token_hex(24)}\nCURSOR_SECRET={secrets.token_hex(32)}\n", encoding="ascii")' +} +docker compose config --quiet +docker compose up -d --build --wait +docker compose ps +Invoke-RestMethod http://127.0.0.1:8080/health/ready +``` -## Доказательства критериев +Сервис `migrate` стартует после PostgreSQL и применяет миграции. Его нормальное состояние после работы — `Exited (0)`. API ждёт успешной миграции и готовности Redis. `APP_PORT` в `.env` задаёт опубликованный порт на `127.0.0.1`. При его изменении передайте свой адрес через `--base-url` тестам. `POSTGRES_PASSWORD` — локальный пароль БД, `CURSOR_SECRET` — ключ HMAC для курсоров, `SESSION_TTL_SECONDS` — срок новых сессий. Оба секрета остаются в игнорируемом `.env`. Повторный запуск использует прежний файл и тома. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +`docker compose down` останавливает стенд с сохранением данных. Для намеренного сброса только этого учебного проекта предусмотрена команда `docker compose down -v`. Она удаляет данные и все сессии. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура -## Эксперименты и отказы +- Из первой лабы остались HTTP-маршруты, проверки запросов, формат ошибок, проверка участников чата и пагинация. При регистрации появился пароль. Для входа используется Basic, после входа клиент передаёт Bearer-токен. Хеш пароля считается через `hashlib.scrypt` с отдельной 16-байтной солью (`N=32768, r=8, p=3`). +- В PostgreSQL лежат пользователи, чаты, участники и сообщения. За уникальность логина отвечает `UNIQUE (username)`. Внешний ключ сообщения проверяет, что отправитель входит в чат. Чат и участники записываются в одной транзакции. Если одного из пользователей нет, откатывается и запись самого чата. API возвращает 201 после `COMMIT`. +- В Redis лежат сессии. Ключ вычисляется как SHA-256 случайного 256-битного токена, значение — UUID пользователя. При входе Redis получает срок истечения через `PXAT`, при выходе ключ удаляется. Сессии и их сроки переживают рестарт благодаря AOF, `appendfsync always` и тому `redis_data`. Очистка Redis приведёт к повторному входу. +- Параметры SQL передаются отдельно от текста запроса. Пул PostgreSQL использует 1–8 соединений, максимум 16 ожидающих и ожидание до 2 секунд. Таймаут подключения — 2 секунды, SQL — 3 секунды, блокировки — 1,5 секунды. Для Redis ограничены соединения (16), подключение и чтение (по 1 секунде), повторы отключены. В контейнере API задана одна попытка DNS с таймаутом 1 секунду. При потере зависимости API возвращает 503 для пути, который её требует. Для защищённого запроса без Bearer ответ остаётся 401. +- История сортируется по `(created_at, id)`. Индекс `messages_history (chat_id, created_at, id)` обслуживает равенство по чату, диапазон после курсора и сортировку. Запрашивается `limit + 1` строка, чтобы узнать о следующей странице. Курсор подписан и привязан к чату, а для списка чатов — к пользователю. Следующее чтение видит подтверждённую запись. При конкурентных вставках запись, чей ключ оказался раньше уже пройденной позиции, может появиться только при новом обходе. Часы влияют на порядок, UUID разрешает совпадения времён. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Проверки -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +```powershell +python scripts/check_contract.py +python tests/smoke.py +docker compose run --rm migrate -## Ограничения и следующие шаги +# Снять StartedAt контейнеров до и после теста +$labIds = docker compose ps -q api postgres redis +docker inspect --format '{{.Name}} {{.State.StartedAt}}' $labIds +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +docker inspect --format '{{.Name}} {{.State.StartedAt}}' $labIds -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +python student-tests/grade4.py +Get-Content -Raw scripts/explain-history.sql | docker compose exec -T postgres psql -X -U messenger -d messenger +``` + +На Linux публичные проверки выполняются через `make check`, `make test` и `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. Если при переносе в форк потерялся бит исполнения, выполните `chmod +x scripts/restart.sh` или добавьте файл через `git add --chmod=+x scripts/restart.sh`. + +Для expiry временно пересоздаём только API. После проверки возвращаем обычный TTL. У существующих сессий сохраняется исходный срок. + +```powershell +$env:SESSION_TTL_SECONDS = '2' +try { + docker compose up -d --no-deps --force-recreate --wait api + python student-tests/grade4.py --expiry-only +} finally { + Remove-Item Env:SESSION_TTL_SECONDS + docker compose up -d --no-deps --force-recreate --wait api +} +``` + +| Критерий | Как проверяется | +| --- | --- | +| Регистрация, доступ, чаты, сообщения и пагинация | Публичный `tests/smoke.py` регистрирует пользователей, входит, отправляет сообщения, обходит историю и проверяет отказ постороннему. | +| Данные и сессия после полного рестарта | `tests/resilience.py` пишет сообщение, выполняет restart hook, повторяет запросы с прежним токеном. `StartedAt` показывает перезапуск API, PostgreSQL и Redis. | +| Повторяемые миграции | Повторный `docker compose run --rm migrate` сверяет реестр версий. | +| Гонка и атомарность | Шесть одновременных регистраций одного имени дают один 201 и пять 409. Неизвестный участник даёт 400 и ноль строк с тестовым названием чата в БД. | +| Redis, logout, ошибки зависимостей | Тест проверяет TTL ключа, 401 при неверном пароле и после logout. По очереди останавливает Redis и PostgreSQL. Проверяет 503 для readiness и действительной сессии и 401 без Bearer. После запуска проверяет прежний токен. | +| Истечение сессии | При TTL 2 секунды один токен даёт сначала 200, после срока — 401. | +| План истории | `scripts/explain-history.sql` вставляет 10 000 строк через `generate_series`, обновляет статистику, снимает `EXPLAIN (ANALYZE, BUFFERS)` первой страницы и страницы после 9000-й записи. Синтетические строки откатываются. | + +## Наблюдаемые результаты + +На Compose-стенде под Windows первый прогон миграций применил `001_initial`, второй вернул `Schema is up to date`. Публичный smoke и проверка полного рестарта прошли. После рестарта изменилось время запуска всех трёх контейнеров, а ранее выданный токен прочитал сохранённое сообщение. [Вывод проверок](evidence/compose-checks.txt) + +Шесть одновременных регистраций дали один 201 и пять 409. После отката чата с неизвестным участником в PostgreSQL осталось ноль строк с его названием. Тест подтвердил короткий TTL, отзыв сессии и сохранение отзыва после перезапуска Redis. При остановке любой зависимости readiness и запрос с действующей сессией вернули 503. Запрос с токеном получил 503 за 2,012 с при остановке Redis и за 2,011 с при остановке PostgreSQL. После запуска зависимости прежний токен снова работал. [Вывод проверок](evidence/compose-checks.txt) + +На синтетических 10 000 сообщениях обе страницы получили по 51 строке через `Index Scan using messages_history`. Первая страница затронула три буфера и заняла 0,073 мс. Глубокая страница после 9000-й строки затронула три буфера и заняла 0,185 мс. У глубокой страницы сравнение пары `(created_at, id)` находится в `Index Cond`. Это один локальный прогон, отражающий форму плана и фактическое число строк. [Вывод EXPLAIN](evidence/history-plan.txt) + +## Границы эксперимента + +Стенд рассчитан на локальные синтетические данные. PostgreSQL и Redis представлены по одному экземпляру. Тесты проверяют обычный рестарт с сохранёнными томами и временный отказ зависимостей. Потеря тома, отказ диска и восстановление на другом хосте в этот сценарий не входят. + +## Порядок демонстрации + +1. Открыть терминал в папке проекта, запустить Compose и показать `docker compose ps` и `/health/ready`. +2. Войти с Basic и отправить сообщение с Bearer по командам ниже. +3. Запустить тест перезапуска и сравнить `StartedAt` трёх контейнеров. Тест прочитает сообщение с прежним токеном. +4. Показать план страницы после 9000-й записи. Условие `ROW(created_at, id) > ...` стоит в `Index Cond`, поэтому PostgreSQL начинает чтение сразу после курсора. +5. Запустить `grade4.py`. Сравнить ответы 401 без действующей сессии и 503 при отказе зависимости. + +У PostgreSQL включены `synchronous_commit` и `fsync`. При этих настройках сервер подтверждает `COMMIT` после записи WAL на диск. Redis восстанавливает сессии из AOF. + +### Ручной вход и сообщение в PowerShell + +```powershell +$labApi = 'http://127.0.0.1:8080/api/v1' +$labName = 'demo_' + [guid]::NewGuid().ToString('N').Substring(0, 12) +$labPass = 'Demo:' + [guid]::NewGuid().ToString('N') +$labRegistration = @{username=$labName; display_name='Демо'; password=$labPass} | ConvertTo-Json +$labUser = Invoke-RestMethod -Method Post -Uri "$labApi/users" -ContentType 'application/json; charset=utf-8' -Body $labRegistration +$labBasic = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes("${labName}:$labPass")) +$labSession = Invoke-RestMethod -Method Post -Uri "$labApi/auth/sessions" -Headers @{Authorization="Basic $labBasic"} +$labAuth = @{Authorization="Bearer $($labSession.token)"} +Invoke-RestMethod -Uri "$labApi/users/me" -Headers $labAuth +$labChatJson = @{title='Чат для защиты'; member_ids=@($labUser.id)} | ConvertTo-Json +$labChat = Invoke-RestMethod -Method Post -Uri "$labApi/chats" -Headers $labAuth -ContentType 'application/json; charset=utf-8' -Body $labChatJson +$labHistory = "$labApi/chats/$($labChat.id)/messages" +$labMessage = @{text='Привет, сообщение сохранено!'} | ConvertTo-Json +Invoke-RestMethod -Method Post -Uri $labHistory -Headers $labAuth -ContentType 'application/json; charset=utf-8' -Body $labMessage +Invoke-RestMethod -Uri $labHistory -Headers $labAuth | ConvertTo-Json -Depth 4 +``` + +## Источники и помощь ИИ + +Основа работы — каркас курса и код первой лабы. Для реализации, тестов и отчёта использовался Codex. + +Документация по трём компонентам. [Индексы PostgreSQL](https://www.postgresql.org/docs/17/indexes-ordering.html), [пул Psycopg](https://www.psycopg.org/psycopg3/docs/advanced/pool.html), [AOF в Redis](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f9244bff --- /dev/null +++ b/compose.yaml @@ -0,0 +1,59 @@ +services: + postgres: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Generate .env as described in REPORT.md} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] + interval: 2s + timeout: 2s + retries: 15 + + redis: + image: redis:7.4.6-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", ""] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 15 + + migrate: + image: labkos-two-api:lab2 + build: . + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Generate .env as described in REPORT.md} + command: ["python", "-m", "messenger.migrate"] + healthcheck: + disable: true + depends_on: + postgres: + condition: service_healthy + + api: + image: labkos-two-api:lab2 + build: . + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Generate .env as described in REPORT.md} + CURSOR_SECRET: ${CURSOR_SECRET:?Generate .env as described in REPORT.md} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + dns_opt: + - timeout:1 + - attempts:1 + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + +volumes: + postgres_data: + redis_data: diff --git a/evidence/compose-checks.txt b/evidence/compose-checks.txt new file mode 100644 index 00000000..78e5ba1f --- /dev/null +++ b/evidence/compose-checks.txt @@ -0,0 +1,43 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +labkos_two-api-1 labkos-two-api:lab2 "python -m messenger" api 9 seconds ago Up 7 seconds (healthy) 127.0.0.1:8080->8080/tcp +labkos_two-migrate-1 labkos-two-api:lab2 "python -m messenger…" migrate 2 minutes ago Exited (0) 2 minutes ago +labkos_two-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 2 minutes ago Up About a minute (healthy) 5432/tcp +labkos_two-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 2 minutes ago Up About a minute (healthy) 6379/tcp +Applied: 001_initial +Schema is up to date +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +Before full restart (UTC) +/labkos_two-api-1 2026-11-05T22:42:13.724276004Z +/labkos_two-postgres-1 2026-11-05T22:40:53.482741183Z +/labkos_two-redis-1 2026-11-05T22:40:51.310589866Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +After full restart (UTC) +/labkos_two-api-1 2026-11-05T22:42:39.6528577Z +/labkos_two-postgres-1 2026-11-05T22:42:39.378949085Z +/labkos_two-redis-1 2026-11-05T22:42:39.28935528Z +appendonly +yes +appendfsync +always + synchronous_commit +-------------------- + on +(1 row) + + fsync +------- + on +(1 row) + +PASS registration race (6 concurrent attempts): [201, 409, 409, 409, 409, 409] +PASS Basic login with a Unicode password containing a colon +PASS rollback: HTTP 400 and zero rows for rejected chat in PostgreSQL +PASS wrong password=401, logout=204, revoked token=401 +PASS Redis session key has TTL (3597s left) +PASS redis outage: live=200 ready=503 Bearer=503 (2.012s) anonymous=401; same token after start (0.060s from docker compose start return) +PASS Redis restart did not revive revoked session +PASS postgres outage: live=200 ready=503 Bearer=503 (2.011s) anonymous=401; same token after start (0.088s from docker compose start return) +PASS 2s expiry: same token 200 before deadline and 401 after deadline +PASS manual PowerShell login, chat, message and history from REPORT.md diff --git a/evidence/history-plan.txt b/evidence/history-plan.txt new file mode 100644 index 00000000..96a58443 --- /dev/null +++ b/evidence/history-plan.txt @@ -0,0 +1,38 @@ +BEGIN +INSERT 0 1 +INSERT 0 1 +INSERT 0 1 +INSERT 0 10000 +ANALYZE + messages_in_test_chat +----------------------- + 10000 +(1 row) + +First page (50 items plus one lookahead) + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.29..4.28 rows=51 width=78) (actual time=0.024..0.044 rows=51 loops=1) + Buffers: shared hit=3 + -> Index Scan using messages_history on messages (cost=0.29..782.85 rows=10000 width=78) (actual time=0.022..0.035 rows=51 loops=1) + Index Cond: (chat_id = ''::uuid) + Buffers: shared hit=3 + Planning: + Buffers: shared hit=18 + Planning Time: 0.268 ms + Execution Time: 0.073 ms +(9 rows) + +Deep page (after message 9000) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (cost=0.29..15.36 rows=51 width=78) (actual time=0.081..0.103 rows=51 loops=1) + Buffers: shared hit=3 + -> Index Scan using messages_history on messages (cost=0.29..296.39 rows=1002 width=78) (actual time=0.079..0.094 rows=51 loops=1) + Index Cond: ((chat_id = ''::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 02:30:00+00'::timestamp with time zone, ''::uuid))) + Buffers: shared hit=3 + Planning Time: 0.178 ms + Execution Time: 0.185 ms +(7 rows) + +ROLLBACK diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..0280fec4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.2.6 +redis==6.4.0 diff --git a/scripts/explain-history.sql b/scripts/explain-history.sql new file mode 100644 index 00000000..5270d9bd --- /dev/null +++ b/scripts/explain-history.sql @@ -0,0 +1,33 @@ +\set ON_ERROR_STOP on +BEGIN; + +INSERT INTO users (username, display_name, password_salt, password_hash) +VALUES ('explain_' || left(replace(gen_random_uuid()::text, '-', ''), 24), + 'EXPLAIN fixture', decode(repeat('00', 16), 'hex'), decode(repeat('00', 32), 'hex')) +RETURNING id AS fixture_user \gset + +INSERT INTO chats (title) VALUES ('EXPLAIN fixture') RETURNING id AS fixture_chat \gset +INSERT INTO chat_members (chat_id, user_id, position) VALUES (:'fixture_chat', :'fixture_user', 1); +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT :'fixture_chat', :'fixture_user', 'synthetic message ' || n, + TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * INTERVAL '1 second' +FROM generate_series(1, 10000) n; +ANALYZE messages; +SELECT count(*) AS messages_in_test_chat FROM messages WHERE chat_id = :'fixture_chat'; + +\echo First page (50 items plus one lookahead) +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = :'fixture_chat' +ORDER BY created_at, id LIMIT 51; + +SELECT created_at AS cursor_time, id AS cursor_id FROM messages +WHERE chat_id = :'fixture_chat' AND created_at = TIMESTAMPTZ '2026-01-01 02:30:00+00' +\gset + +\echo Deep page (after message 9000) +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = :'fixture_chat' + AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..9d3b25cf --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +docker compose --project-directory "%~dp0.." restart postgres redis api diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..f551cde0 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec docker compose restart postgres redis api diff --git a/src/healthcheck.py b/src/healthcheck.py new file mode 100644 index 00000000..a701b10d --- /dev/null +++ b/src/healthcheck.py @@ -0,0 +1,12 @@ +"""Docker HEALTHCHECK probe: exit 0 if /health/ready answers 200, else 1.""" + +import os +import sys +import urllib.request + +port = os.environ.get("APP_PORT", "8080") +try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response: + sys.exit(0 if response.status == 200 else 1) +except Exception: + sys.exit(1) diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..bbfe9c32 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,3 @@ +"""PostgreSQL and Redis messenger for MTUSI Lab 2.""" + +__version__ = "2.0.0" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..3878e005 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,58 @@ +"""Run the Lab 2 HTTP API.""" + +from __future__ import annotations + +import os +import signal +import sys +import threading +import time + +from . import __version__ +from .config import Config +from .http_api import MessengerServer +from .limits import Limits +from .logs import log +from .sessions import Sessions +from .store import Store + + +def main() -> None: + config = Config.from_env() + limits = Limits() + store = Store(config.cursor_secret) + sessions = Sessions(config.session_ttl) + server = MessengerServer((config.host, config.port), store, sessions, limits, config.max_body_bytes) + stop_requested = threading.Event() + + def on_signal(signum: int, _frame) -> None: + if stop_requested.is_set(): + return + stop_requested.set() + server.draining = True # readiness -> 503, new API requests -> 503 + log("info", "shutdown_started", signal=signal.Signals(signum).name) + # shutdown() waits for serve_forever() to return, so it must run in another thread. + threading.Thread(target=server.shutdown, name="shutdown", daemon=True).start() + + signal.signal(signal.SIGTERM, on_signal) + signal.signal(signal.SIGINT, on_signal) + + log("info", "started", version=__version__, host=config.host, port=config.port, env=config.env, + pid=os.getpid(), storage="postgres", session_store="redis", max_body_bytes=config.max_body_bytes, + **limits.describe()) + limits.start_sampler() + server.serve_forever(poll_interval=0.2) + + stopped_at = time.monotonic() + server.server_close() # stop accepting connections: the listening socket is closed + finished = server.wait_idle(timeout=config.shutdown_timeout) + store.pool.close() + sessions.redis.close() + log("info" if finished else "warning", "shutdown_complete", + in_flight_finished=finished, + drain_ms=round((time.monotonic() - stopped_at) * 1000, 1)) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..b92a4216 --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,68 @@ +"""Local configuration from environment variables.""" + +from __future__ import annotations + +import os +import sys +from dataclasses import dataclass + +from .logs import log + + +def postgres_options() -> dict: + return { + "host": os.environ.get("POSTGRES_HOST", "postgres"), + "port": env_int("POSTGRES_PORT", 5432, 1, 65535), + "dbname": os.environ.get("POSTGRES_DB", "messenger"), + "user": os.environ.get("POSTGRES_USER", "messenger"), + "password": os.environ.get("POSTGRES_PASSWORD"), + "connect_timeout": 2, + "options": "-c statement_timeout=3000 -c lock_timeout=1500 -c timezone=UTC", + } + + +def env_int(name: str, default: int, low: int, high: int) -> int: + raw = os.environ.get(name, "").strip() or str(default) + if not raw.isdigit() or not low <= int(raw) <= high: + log("error", "bad_config", variable=name, allowed=f"{low}..{high}") + sys.exit(2) + return int(raw) + + +def env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name, "").strip().lower() + if not raw: + return default + if raw in ("1", "true", "yes", "on"): + return True + if raw in ("0", "false", "no", "off"): + return False + log("error", "bad_config", variable=name, allowed="0/1") + sys.exit(2) + + +@dataclass(frozen=True) +class Config: + host: str + port: int + env: str + shutdown_timeout: int + max_body_bytes: int + session_ttl: int + cursor_secret: bytes + + @classmethod + def from_env(cls) -> "Config": + secret = os.environ.get("CURSOR_SECRET", "").encode() + if len(secret) < 32: + log("error", "bad_config", variable="CURSOR_SECRET", minimum_length=32) + sys.exit(2) + return cls( + host=os.environ.get("APP_HOST", "").strip() or "0.0.0.0", + port=env_int("APP_PORT", 8080, 1, 65535), + env=os.environ.get("APP_ENV", "lab"), + shutdown_timeout=env_int("SHUTDOWN_TIMEOUT_SECONDS", 8, 1, 9), + max_body_bytes=env_int("MAX_BODY_BYTES", 64 * 1024, 1024, 16 * 1024 * 1024), + session_ttl=env_int("SESSION_TTL_SECONDS", 3600, 1, 31_536_000), + cursor_secret=secret, + ) diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..9822a567 --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,68 @@ +"""API errors in the common contract format: {"error": {"code", "message", "request_id"}}.""" + +from __future__ import annotations + +from typing import Optional + + +class ApiError(Exception): + """An error that is turned into a JSON error response.""" + + def __init__(self, status: int, code: str, message: str, close: bool = False, + retry_after: Optional[int] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + # close=True: the request body may not have been read, so the connection + # must not be reused for another request. + self.close = close + self.retry_after = retry_after # seconds, sent as the Retry-After header + + +def bad_request(message: str, close: bool = False) -> ApiError: + return ApiError(400, "invalid_request", message, close) + + +def unauthorized() -> ApiError: + return ApiError(401, "unauthorized", "invalid credentials or session") + + +def not_found(message: str = "resource not found") -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message, close=True) + + +def unavailable(message: str) -> ApiError: + return ApiError(503, "unavailable", message) + + +def internal_error() -> ApiError: + # Never expose stack traces or internal details to the client. + return ApiError(500, "internal_error", "internal server error") + + +_CODE_BY_STATUS = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", +} + + +def code_for_status(status: int) -> str: + """Contract error code for an arbitrary HTTP status (used for low-level HTTP errors).""" + if status in _CODE_BY_STATUS: + return _CODE_BY_STATUS[status] + return "internal_error" if status >= 500 else "invalid_request" diff --git a/src/messenger/http_api.py b/src/messenger/http_api.py new file mode 100644 index 00000000..efa2c07a --- /dev/null +++ b/src/messenger/http_api.py @@ -0,0 +1,432 @@ +"""HTTP layer: routing, session authentication, JSON responses, request logs.""" + +from __future__ import annotations + +import json +import re +import secrets +import socketserver +import threading +import time +import uuid +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Callable, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, unquote, urlsplit + +from . import validation as v +from .errors import ApiError, bad_request, code_for_status, internal_error, not_found, unauthorized, unavailable +from .limits import Limits +from .logs import log +from .sessions import Sessions, credentials, password_digest, verify_password +from .store import Store + +REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class MessengerServer(ThreadingHTTPServer): + """Thread per connection with bounded admission and graceful shutdown.""" + + daemon_threads = True # idle keep-alive connections must not block process exit + request_queue_size = 128 + + def __init__(self, address: Tuple[str, int], store: Store, sessions: Sessions, + limits: Limits, max_body_bytes: int): + self.store = store + self.sessions = sessions + self.limits = limits + self.max_body_bytes = max_body_bytes + self.draining = False + self._in_flight = 0 + self._connections = 0 + self._idle = threading.Condition() + super().__init__(address, Handler) + + def server_bind(self) -> None: + # HTTPServer.server_bind() does a reverse DNS lookup (getfqdn) that can + # hang for seconds inside containers; it is not needed for an API. + socketserver.TCPServer.server_bind(self) + self.server_name, self.server_port = self.server_address[:2] + + # -- connections: one thread each; refuse new ones above MAX_CONNECTIONS -- + + def process_request(self, request, client_address) -> None: + with self._idle: + refuse = self.limits.enabled and self._connections >= self.limits.max_connections + if not refuse: + self._connections += 1 + if refuse: + _refuse_connection(request) + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + with self._idle: + self._connections -= 1 + raise + + def process_request_thread(self, request, client_address) -> None: + try: + super().process_request_thread(request, client_address) + finally: + with self._idle: + self._connections -= 1 + + # -- requests in progress -- + + def request_started(self) -> int: + with self._idle: + self._in_flight += 1 + return self._in_flight + + def request_finished(self) -> None: + with self._idle: + self._in_flight -= 1 + if self._in_flight == 0: + self._idle.notify_all() + + def wait_idle(self, timeout: float) -> bool: + """Wait until no request is being processed; False on timeout.""" + with self._idle: + return self._idle.wait_for(lambda: self._in_flight == 0, timeout=max(timeout, 0)) + + +# ----- route handlers -------------------------------------------------------- +# Each handler gets (request, user, path params, query) and returns (status, body). + +Result = Tuple[int, Optional[Dict[str, Any]]] + + +def health_live(req: "Handler", user, params, query) -> Result: + return 200, {"status": "ok"} + + +def _refuse_connection(sock) -> None: + """Answer 503 without starting a thread (connection limit). Runs in the accept loop.""" + request_id = uuid.uuid4().hex + body = json.dumps({"error": {"code": "unavailable", "message": "too many connections, retry later", + "request_id": request_id}}).encode() + head = ( + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\nX-Request-Id: {request_id}\r\nRetry-After: 1\r\n" + "Connection: close\r\n\r\n" + ).encode() + try: + sock.settimeout(0.05) + try: + sock.recv(65536) # read what already arrived so close() does not reset the connection + except OSError: + pass + sock.sendall(head + body) + except OSError: + pass + log("info", "request", request_id=request_id, method="-", route="-", status=503, + duration_ms=0.0, rejected="connections") + + +def health_ready(req: "Handler", user, params, query) -> Result: + if req.server.draining: + raise unavailable("server is shutting down") + if not req.server.limits.ready(): + raise unavailable("memory limit reached") + req.server.store.ready() + req.server.sessions.ready() + return 200, {"status": "ok"} + + +def create_user(req: "Handler", user, params, query) -> Result: + fields = ("username", "display_name", "password") + data = req.json_body(allowed=fields, required=fields) + username = v.username_field(data) + display_name = v.text_field(data, "display_name", 100) + password = data["password"] + if not v.valid_password(password): + raise bad_request("password must contain 12..128 valid nonblank characters") + salt = secrets.token_bytes(16) + return 201, req.server.store.create_user(username, display_name, salt, password_digest(password, salt)) + + +def create_session(req: "Handler", user, params, query) -> Result: + if req._read_body(): + raise bad_request("login does not accept a body") + username, password = credentials(req.authorization("basic")) + user = req.server.store.credentials(username) + if not verify_password(password, user): + raise unauthorized() + del user["password_hash"], user["password_salt"] + token, expires_at = req.server.sessions.create(user["id"]) + return 201, {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +def revoke_session(req: "Handler", user, params, query) -> Result: + req.server.sessions.revoke(req.authorization("bearer")) + return 204, None + + +def get_me(req: "Handler", user, params, query) -> Result: + profile = req.server.store.get_user(user) + if profile is None: + raise unauthorized() + return 200, profile + + +def create_chat(req: "Handler", user, params, query) -> Result: + req.server.limits.check_rate(user) # 429 if this user writes too fast + data = req.json_body(allowed=("title", "member_ids"), required=("title", "member_ids")) + title = v.text_field(data, "title", 100) + members = v.member_ids_field(data) + return 201, req.server.store.create_chat(user, title, members) + + +def list_chats(req: "Handler", user, params, query) -> Result: + limit, cursor = v.limit_param(query), v.cursor_param(query) + return 200, req.server.store.list_chats(user, limit, cursor) + + +def get_chat(req: "Handler", user, params, query) -> Result: + return 200, req.server.store.get_chat(user, params["chat_id"]) + + +def post_message(req: "Handler", user, params, query) -> Result: + req.server.limits.check_rate(user) # 429 if this user writes too fast + # sender_id is not accepted in the body: the sender is always the current user. + data = req.json_body(allowed=("text",), required=("text",)) + text = v.text_field(data, "text", 4000) + return 201, req.server.store.add_message(user, params["chat_id"], text) + + +def list_messages(req: "Handler", user, params, query) -> Result: + limit, cursor = v.limit_param(query), v.cursor_param(query) + return 200, req.server.store.list_messages(user, params["chat_id"], limit, cursor) + + +def json_value(value): + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + raise TypeError(f"Cannot serialize {type(value).__name__}") + + +Route = Tuple[str, "re.Pattern[str]", str, Callable[..., Result], bool] + +_CHAT = r"(?P[^/]+)" +ROUTES: List[Route] = [ + # method, path regex, template for logs, handler, requires Bearer + ("GET", re.compile(r"/health/live"), "/health/live", health_live, False), + ("GET", re.compile(r"/health/ready"), "/health/ready", health_ready, False), + ("POST", re.compile(r"/api/v1/users"), "/api/v1/users", create_user, False), + ("POST", re.compile(r"/api/v1/auth/sessions"), "/api/v1/auth/sessions", create_session, False), + ("DELETE", re.compile(r"/api/v1/auth/sessions/current"), "/api/v1/auth/sessions/current", revoke_session, False), + ("GET", re.compile(r"/api/v1/users/me"), "/api/v1/users/me", get_me, True), + ("POST", re.compile(r"/api/v1/chats"), "/api/v1/chats", create_chat, True), + ("GET", re.compile(r"/api/v1/chats"), "/api/v1/chats", list_chats, True), + ("GET", re.compile(r"/api/v1/chats/" + _CHAT), "/api/v1/chats/{chat_id}", get_chat, True), + ("POST", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", post_message, True), + ("GET", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", list_messages, True), +] + + +def resolve(method: str, path: str) -> Tuple[Route, Dict[str, str]]: + path_matched = False + for route in ROUTES: + match = route[1].fullmatch(path) + if match is None: + continue + path_matched = True + if route[0] == method or (method == "HEAD" and route[0] == "GET"): + return route, match.groupdict() + if path_matched: + raise ApiError(405, "invalid_request", "method not allowed") + raise not_found("route not found") + + +# ----- HTTP handler ---------------------------------------------------------- + + +class Handler(BaseHTTPRequestHandler): + server: MessengerServer + protocol_version = "HTTP/1.1" + server_version = "mtusi-messenger" + sys_version = "" + timeout = 30 # idle keep-alive connections are closed after this many seconds + # TCP_NODELAY: headers and body are written separately; with Nagle's algorithm plus the + # client's delayed ACK every keep-alive response waited ~40 ms (found in the track 2 runs). + disable_nagle_algorithm = True + + request_id = "" + route_template = "-" + started = 0.0 + _body: Optional[bytes] = None + + def do_GET(self) -> None: + self._handle() + + def do_POST(self) -> None: + self._handle() + + def do_PUT(self) -> None: + self._handle() + + def do_PATCH(self) -> None: + self._handle() + + def do_DELETE(self) -> None: + self._handle() + + def do_HEAD(self) -> None: + self._handle() + + def do_OPTIONS(self) -> None: + self._handle() + + # -- request processing -- + + def _handle(self) -> None: + self.started = time.monotonic() + incoming = self.headers.get("X-Request-Id", "") + self.request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex + self.route_template = "-" + self._body: Optional[bytes] = None + self.server.request_started() + slot = False + try: + if self.command in ("POST", "PUT", "PATCH", "DELETE"): + # Read the body first: an early error then never leaves unread bytes + # on the connection, so keep-alive works even for 429/503 answers. + self._read_body() + split = urlsplit(self.path) + route, params = resolve(self.command, split.path) + self.route_template = route[2] + is_health = route[2] in ("/health/live", "/health/ready") + if self.server.draining and not is_health: + raise unavailable("server is shutting down") + if not is_health: + slot = self.server.limits.acquire_slot() # bounded queue; 503 when overloaded + user = self.server.sessions.identify(self.authorization("bearer")) if route[4] else None + if "chat_id" in params: + params["chat_id"] = v.parse_uuid(unquote(params["chat_id"]), "chat_id") + query = parse_qs(split.query, keep_blank_values=True) + status, payload = route[3](self, user, params, query) + self._send_json(status, payload) + except ApiError as error: + if error.close: + self.close_connection = True + self._safe_send_error(error) + except (BrokenPipeError, ConnectionResetError, TimeoutError): + self.close_connection = True # the client went away; nothing to answer + except Exception as error: + log("error", "unhandled_exception", request_id=self.request_id, + route=self.route_template, exception_type=type(error).__name__) + self.close_connection = True + self._safe_send_error(internal_error()) + finally: + if slot: + self.server.limits.release_slot() + self.server.request_finished() + + def _safe_send_error(self, error: ApiError) -> None: + try: + self._send_error_json(error) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + + def authorization(self, scheme: str) -> str: + values = self.headers.get_all("Authorization") or [] + if len(values) != 1: + raise unauthorized() + parts = values[0].split() + if len(parts) != 2 or parts[0].lower() != scheme: + raise unauthorized() + return parts[1] + + def _read_body(self) -> bytes: + if self._body is not None: + return self._body + if self.headers.get("Transfer-Encoding"): + raise bad_request("chunked request bodies are not supported", close=True) + lengths = self.headers.get_all("Content-Length") or [] + if not lengths: + self._body = b"" + return self._body + if len(lengths) != 1 or not re.fullmatch(r"[0-9]{1,16}", lengths[0]): + raise bad_request("invalid Content-Length", close=True) + length = int(lengths[0]) + if length > self.server.max_body_bytes: + raise ApiError(413, "payload_too_large", "request body is too large", close=True) + self._body = self.rfile.read(length) + if len(self._body) != length: + raise bad_request("incomplete request body", close=True) + return self._body + + def json_body(self, allowed, required) -> Dict[str, Any]: + raw = self._read_body() + if self.headers.get_content_type() != "application/json": + raise bad_request("Content-Type must be application/json") + charset = self.headers.get_content_charset() + if charset not in (None, "utf-8"): + raise bad_request("request body must be UTF-8") + return v.parse_json_object(raw, allowed, required) + + # -- responses -- + + def _send_json(self, status: int, payload: Optional[Dict[str, Any]], retry_after: Optional[int] = None) -> None: + body = b"" if payload is None else json.dumps( + payload, ensure_ascii=False, separators=(",", ":"), default=json_value + ).encode("utf-8") + if self.command in ("POST", "PUT", "PATCH", "DELETE") and self._body is None: + # The body was never read (e.g. error before parsing): do not reuse the connection. + self.close_connection = True + if self.server.draining: + self.close_connection = True + self.send_response(status) + if status != 204: + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", self.request_id) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + if retry_after is not None: + self.send_header("Retry-After", str(retry_after)) + if status == 401: + challenge = 'Bearer realm="messenger"' + if self.route_template == "/api/v1/auth/sessions": + challenge = 'Basic realm="messenger", charset="UTF-8"' + self.send_header("WWW-Authenticate", challenge) + if status == 405: + path = urlsplit(getattr(self, "path", "") or "").path + allow = sorted({r[0] for r in ROUTES if r[1].fullmatch(path)}) + if allow: + self.send_header("Allow", ", ".join(allow)) + if self.close_connection: + self.send_header("Connection", "close") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + log("info", "request", request_id=self.request_id, method=self.command, + route=self.route_template, status=status, + duration_ms=round((time.monotonic() - self.started) * 1000, 2)) + + def _send_error_json(self, error: ApiError) -> None: + self._send_json(error.status, { + "error": {"code": error.code, "message": error.message, "request_id": self.request_id} + }, retry_after=error.retry_after) + + def send_error(self, code: int, message: Optional[str] = None, explain: Optional[str] = None) -> None: + """Low-level HTTP errors (bad request line, unsupported method) in the same JSON format.""" + self.request_id = uuid.uuid4().hex + self.started = time.monotonic() + self.route_template = "-" + self._body = None + self.close_connection = True + if getattr(self, "command", None) is None: + self.command = "GET" + status = 405 if code == 501 else code # 501 = method unknown to http.server + self._safe_send_error(ApiError(status, code_for_status(status), "invalid HTTP request")) + + def version_string(self) -> str: + return self.server_version + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + pass # replaced by structured logs in _send_json diff --git a/src/messenger/limits.py b/src/messenger/limits.py new file mode 100644 index 00000000..45860f5c --- /dev/null +++ b/src/messenger/limits.py @@ -0,0 +1,166 @@ +"""Bound HTTP connections, concurrent work and writes per user.""" + +from __future__ import annotations + +import math +import os +import threading +import time +from typing import Dict, List, Optional +from uuid import UUID + +from .config import env_bool, env_int +from .errors import ApiError + +try: + _PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") +except (AttributeError, ValueError, OSError): + _PAGE_SIZE = 4096 + + +def read_rss() -> Optional[int]: + """Resident memory of this process in bytes (Linux /proc), None elsewhere.""" + try: + with open("/proc/self/statm") as f: + return int(f.read().split()[1]) * _PAGE_SIZE + except (OSError, ValueError, IndexError): + return None + + +def _read_limit(path: str) -> Optional[int]: + try: + with open(path) as f: + raw = f.read().strip() + except OSError: + return None + if not raw.isdigit(): + return None # cgroup v2 says "max" when there is no limit + value = int(raw) + return value if value < (1 << 50) else None # cgroup v1 uses a huge number for "no limit" + + +def detect_memory_limit() -> Optional[int]: + """Memory limit of the container (cgroup v2 or v1), None if unlimited/unknown.""" + candidates: List[str] = ["/sys/fs/cgroup/memory.max"] + try: + with open("/proc/self/cgroup") as f: + for line in f: + _, controllers, path = line.rstrip("\n").split(":", 2) + if controllers == "": + candidates.append(f"/sys/fs/cgroup{path}/memory.max") + elif "memory" in controllers.split(","): + candidates.append(f"/sys/fs/cgroup/memory{path}/memory.limit_in_bytes") + except (OSError, ValueError): + pass + candidates.append("/sys/fs/cgroup/memory/memory.limit_in_bytes") + for path in candidates: + value = _read_limit(path) + if value: + return value + return None + + +class Limits: + def __init__(self) -> None: + self.enabled = env_bool("LIMITS_ENABLED", True) + # Bounded request queue: MAX_IN_FLIGHT requests are processed at once, up to + # MAX_QUEUE more wait for a slot at most QUEUE_TIMEOUT_MS; the rest get 503 at once. + self.max_in_flight = env_int("MAX_IN_FLIGHT", 8, 1, 10_000) + self.max_queue = env_int("MAX_QUEUE", 64, 0, 100_000) + self.queue_timeout = env_int("QUEUE_TIMEOUT_MS", 1000, 1, 60_000) / 1000 + self._slots = threading.BoundedSemaphore(self.max_in_flight) + self.waiting = 0 + self.max_connections = env_int("MAX_CONNECTIONS", 256, 1, 100_000) + self.user_write_rate = env_int("USER_WRITE_RATE", 50, 1, 100_000) # tokens per second + self.user_write_burst = env_int("USER_WRITE_BURST", 100, 1, 100_000) + configured = env_int("MEMORY_LIMIT_BYTES", 0, 0, 1 << 50) + self.memory_limit = configured or detect_memory_limit() + self.hard_percent = env_int("MEMORY_HARD_PERCENT", 90, 10, 99) + self.rss = read_rss() or 0 + self._lock = threading.Lock() + self._buckets: Dict[UUID, List[float]] = {} + + # ----- description / stats ------------------------------------------------ + + def describe(self) -> Dict[str, object]: + return { + "limits_enabled": self.enabled, + "max_in_flight": self.max_in_flight, + "max_queue": self.max_queue, + "queue_timeout_ms": int(self.queue_timeout * 1000), + "max_connections": self.max_connections, + "user_write_rate": self.user_write_rate, + "user_write_burst": self.user_write_burst, + "memory_limit_bytes": self.memory_limit, + "memory_hard_percent": self.hard_percent, + } + + def _over(self, percent: int) -> bool: + return bool(self.enabled and self.memory_limit and self.rss >= self.memory_limit * percent / 100) + + # ----- checks (each raises ApiError when the limit is exceeded) ------------ + + def acquire_slot(self) -> bool: + """Admission for every non-health request, before any work is done. + + Returns True when a processing slot was taken (the caller must call + release_slot()), False when limits are off. Raises 503 when overloaded. + """ + if not self.enabled: + return False + if self._over(self.hard_percent): + raise ApiError(503, "unavailable", "memory limit reached, retry later", retry_after=5) + if self._slots.acquire(blocking=False): + return True + with self._lock: + queue_full = self.waiting >= self.max_queue + if not queue_full: + self.waiting += 1 + if queue_full: + raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1) + try: + acquired = self._slots.acquire(timeout=self.queue_timeout) + finally: + with self._lock: + self.waiting -= 1 + if not acquired: + raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1) + return True + + def release_slot(self) -> None: + self._slots.release() + + def ready(self) -> bool: + return not self._over(self.hard_percent) + + def check_rate(self, user_id: UUID) -> None: + """Token bucket per user for write requests.""" + if not self.enabled: + return + now = time.monotonic() + with self._lock: + bucket = self._buckets.get(user_id) + if bucket is None: + bucket = self._buckets[user_id] = [float(self.user_write_burst), now] + tokens = min(self.user_write_burst, bucket[0] + (now - bucket[1]) * self.user_write_rate) + bucket[1] = now + if tokens >= 1: + bucket[0] = tokens - 1 + return + bucket[0] = tokens + wait = max(1, math.ceil((1 - tokens) / self.user_write_rate)) + raise ApiError(429, "rate_limited", "too many writes, slow down", retry_after=wait) + + # ----- background sampler ------------------------------------------------- + + def start_sampler(self) -> None: + """Refresh RSS for request admission and readiness.""" + + def loop() -> None: + while True: + rss = read_rss() + if rss is not None: + self.rss = rss + time.sleep(0.1) + + threading.Thread(target=loop, name="limits-sampler", daemon=True).start() diff --git a/src/messenger/logs.py b/src/messenger/logs.py new file mode 100644 index 00000000..16cfe779 --- /dev/null +++ b/src/messenger/logs.py @@ -0,0 +1,27 @@ +"""Structured logs: one JSON object per line on stdout. + +Logs never contain message texts, request bodies, query strings or headers. +""" + +from __future__ import annotations + +import json +import sys +import threading +from datetime import datetime, timezone +from typing import Any + +_lock = threading.Lock() + + +def log(level: str, event: str, **fields: Any) -> None: + record = { + "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + "level": level, + "event": event, + **fields, + } + line = json.dumps(record, ensure_ascii=False, default=str) + with _lock: + sys.stdout.write(line + "\n") + sys.stdout.flush() diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..0d79f9a4 --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,31 @@ +"""Apply versioned migrations under one PostgreSQL transaction and lock.""" + +from pathlib import Path + +import psycopg + +from .config import postgres_options + + +def main() -> None: + migrations = Path(__file__).with_name("migrations").glob("*.sql") + with psycopg.connect(**postgres_options()) as conn: + conn.execute("SELECT pg_advisory_xact_lock(202602)") + conn.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + """) + applied = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + pending = [] + for path in sorted(migrations): + if path.stem not in applied: + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.stem,)) + pending.append(path.stem) + print("Applied: " + ", ".join(pending) if pending else "Schema is up to date") + + +if __name__ == "__main__": + main() diff --git a/src/messenger/migrations/001_initial.sql b/src/messenger/migrations/001_initial.sql new file mode 100644 index 00000000..9c298a2c --- /dev/null +++ b/src/messenger/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash bytea NOT NULL, + password_salt bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL CHECK (position BETWEEN 1 AND 100), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id, chat_id); +CREATE INDEX chats_order ON chats (created_at, id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 00000000..534dc1fd --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,45 @@ +"""The lab 1 signed cursor with a stable key and a PostgreSQL sort position.""" + +import base64 +import binascii +import hashlib +import hmac +import json +from datetime import datetime +from uuid import UUID + +from .errors import bad_request + + +class Cursors: + def __init__(self, secret: bytes): + self.secret = secret + + def _sign(self, payload: bytes) -> bytes: + return hmac.new(self.secret, payload, hashlib.sha256).digest()[:16] + + def decode(self, scope: str, cursor: str) -> tuple[datetime, UUID]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.b64decode(padded, altchars=b"-_", validate=True) + except (binascii.Error, ValueError): + raise bad_request("invalid cursor") from None + payload, signature = raw[:-16], raw[-16:] + if not payload or not hmac.compare_digest(signature, self._sign(payload)): + raise bad_request("invalid cursor") + cursor_scope, timestamp, object_id = json.loads(payload) + if cursor_scope != scope: + raise bad_request("cursor belongs to another resource") + return datetime.fromisoformat(timestamp), UUID(object_id) + + def page(self, scope: str, items: list[dict], limit: int) -> dict: + cursor = None + if len(items) > limit: + items = items[:limit] + last = items[-1] + payload = json.dumps( + [scope, last["created_at"].isoformat(), str(last["id"])], + separators=(",", ":"), + ).encode() + cursor = base64.urlsafe_b64encode(payload + self._sign(payload)).decode().rstrip("=") + return {"items": items, "next_cursor": cursor} diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 00000000..f64004ec --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,96 @@ +"""Basic credentials, scrypt password hashes and opaque Redis sessions.""" + +import base64 +import binascii +import hashlib +import hmac +import os +import re +import secrets +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from uuid import UUID + +from redis import Redis +from redis.backoff import NoBackoff +from redis.exceptions import RedisError +from redis.retry import Retry + +from .errors import unauthorized, unavailable +from .validation import USERNAME_RE, valid_password + +TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{43}") + + +def credentials(header: str) -> tuple[str, str]: + try: + decoded = base64.b64decode(header, validate=True).decode("utf-8") + except (binascii.Error, ValueError): + raise unauthorized() from None + username, _, password = decoded.partition(":") + if not USERNAME_RE.fullmatch(username) or not valid_password(password): + raise unauthorized() + return username, password + + +def password_digest(password: str, salt: bytes) -> bytes: + return hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=32768, r=8, p=3, dklen=32, + maxmem=64 * 1024 * 1024, + ) + + +def verify_password(password: str, user: dict | None) -> bool: + # Do equivalent KDF work for unknown usernames. + salt, expected = (user["password_salt"], user["password_hash"]) if user else (bytes(16), bytes(32)) + match = hmac.compare_digest(password_digest(password, salt), expected) + return user is not None and match + + +@contextmanager +def redis_operation(): + try: + yield + except RedisError: + raise unavailable("Redis unavailable") from None + + +class Sessions: + def __init__(self, ttl: int): + self.ttl = ttl + self.redis = Redis.from_url( + os.environ.get("REDIS_URL", "redis://redis:6379/0"), + socket_connect_timeout=1, socket_timeout=1, max_connections=16, + retry=Retry(NoBackoff(), 0), decode_responses=True, + ) + + @staticmethod + def key(token: str) -> str: + if not TOKEN_RE.fullmatch(token): + raise unauthorized() + return "lab2:session:" + hashlib.sha256(token.encode("ascii")).hexdigest() + + def ready(self) -> None: + with redis_operation(): + self.redis.ping() + + def create(self, user_id: UUID) -> tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_ms = time.time_ns() // 1_000_000 + self.ttl * 1000 + with redis_operation(): + if not self.redis.set(self.key(token), str(user_id), pxat=expires_ms, nx=True): + raise unavailable("could not create session") + return token, datetime.fromtimestamp(expires_ms / 1000, tz=timezone.utc) + + def identify(self, token: str) -> UUID: + with redis_operation(): + user_id = self.redis.get(self.key(token)) + if user_id is None: + raise unauthorized() + return UUID(user_id) + + def revoke(self, token: str) -> None: + with redis_operation(): + if self.redis.delete(self.key(token)) == 0: + raise unauthorized() diff --git a/src/messenger/store.py b/src/messenger/store.py new file mode 100644 index 00000000..7c494e17 --- /dev/null +++ b/src/messenger/store.py @@ -0,0 +1,140 @@ +"""PostgreSQL storage for users, chats, membership and message history.""" + +from contextlib import contextmanager +from uuid import UUID + +from psycopg import OperationalError +from psycopg.errors import ForeignKeyViolation, LockNotAvailable, QueryCanceled +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, TooManyRequests + +from .config import postgres_options +from .errors import bad_request, conflict, not_found, unavailable +from .pagination import Cursors + + +class Store: + def __init__(self, cursor_secret: bytes): + self.cursors = Cursors(cursor_secret) + self.pool = ConnectionPool( + kwargs={**postgres_options(), "row_factory": dict_row}, + min_size=1, max_size=8, max_waiting=16, timeout=2, open=False, + ) + self.pool.open() + + @contextmanager + def connection(self): + try: + # Commits on success and rolls back on any exception before reuse. + with self.pool.connection() as conn: + yield conn + except (OperationalError, TooManyRequests, LockNotAvailable, QueryCanceled): + raise unavailable("PostgreSQL unavailable") from None + + def ready(self) -> None: + with self.connection() as conn: + conn.execute("SELECT 1") + + def create_user(self, username: str, display_name: str, salt: bytes, password_hash: bytes) -> dict: + with self.connection() as conn: + user = conn.execute(""" + INSERT INTO users (username, display_name, password_salt, password_hash) + VALUES (%s, %s, %s, %s) + ON CONFLICT (username) DO NOTHING + RETURNING id, username, display_name, created_at + """, (username, display_name, salt, password_hash)).fetchone() + if user is None: + raise conflict("username is already taken") + return user + + def get_user(self, user_id: UUID) -> dict | None: + with self.connection() as conn: + return conn.execute(""" + SELECT id, username, display_name, created_at FROM users WHERE id = %s + """, (user_id,)).fetchone() + + def credentials(self, username: str) -> dict | None: + with self.connection() as conn: + return conn.execute(""" + SELECT id, username, display_name, created_at, password_salt, password_hash + FROM users WHERE username = %s + """, (username,)).fetchone() + + def create_chat(self, user_id: UUID, title: str, member_ids: list[UUID]) -> dict: + if user_id not in member_ids: + raise bad_request("member_ids must include the current user") + try: + with self.connection() as conn: + chat = conn.execute(""" + INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at + """, (title,)).fetchone() + conn.execute(""" + INSERT INTO chat_members (chat_id, user_id, position) + SELECT %s, user_id, position + FROM unnest(%s::uuid[]) WITH ORDINALITY AS members(user_id, position) + """, (chat["id"], member_ids)) + except ForeignKeyViolation: + raise bad_request("member_ids contains an unknown user") from None + return {**chat, "member_ids": member_ids} + + def get_chat(self, user_id: UUID, chat_id: UUID) -> dict: + with self.connection() as conn: + chat = conn.execute(""" + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m + WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids + FROM chat_members member JOIN chats c ON c.id = member.chat_id + WHERE member.user_id = %s AND c.id = %s + """, (user_id, chat_id)).fetchone() + if chat is None: + raise not_found("chat not found") + return chat + + def list_chats(self, user_id: UUID, limit: int, cursor: str | None) -> dict: + scope = f"chats:{user_id}" + query = """ + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m + WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids + FROM chat_members member JOIN chats c ON c.id = member.chat_id + WHERE member.user_id = %s + """ + params = (user_id,) + if cursor is not None: + query += " AND (c.created_at, c.id) > (%s, %s)" + params += self.cursors.decode(scope, cursor) + with self.connection() as conn: + items = conn.execute( + query + " ORDER BY c.created_at, c.id LIMIT %s", (*params, limit + 1) + ).fetchall() + return self.cursors.page(scope, items, limit) + + def add_message(self, user_id: UUID, chat_id: UUID, text: str) -> dict: + with self.connection() as conn: + message = conn.execute(""" + INSERT INTO messages (chat_id, sender_id, text) + SELECT chat_id, user_id, %s FROM chat_members + WHERE chat_id = %s AND user_id = %s + RETURNING id, chat_id, sender_id, text, created_at + """, (text, chat_id, user_id)).fetchone() + if message is None: + raise not_found("chat not found") + return message + + def list_messages(self, user_id: UUID, chat_id: UUID, limit: int, cursor: str | None) -> dict: + scope = f"messages:{chat_id}" + query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s" + params = (chat_id,) + with self.connection() as conn: + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() + if member is None: + raise not_found("chat not found") + if cursor is not None: + query += " AND (created_at, id) > (%s, %s)" + params += self.cursors.decode(scope, cursor) + items = conn.execute( + query + " ORDER BY created_at, id LIMIT %s", (*params, limit + 1) + ).fetchall() + return self.cursors.page(scope, items, limit) diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 00000000..707e343b --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,129 @@ +"""Strict request validation: bodies, fields, path and query parameters.""" + +from __future__ import annotations + +import json +import re +from typing import Any, Dict, Iterable, List, Optional +from uuid import UUID + +from .errors import bad_request + +UUID_RE = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" +) +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +DIGITS_RE = re.compile(r"[0-9]{1,3}") + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 100 +MAX_CURSOR_LENGTH = 512 + + +def is_uuid(value: Any) -> bool: + return isinstance(value, str) and UUID_RE.fullmatch(value) is not None + + +def parse_uuid(value: Any, what: str) -> UUID: + if not is_uuid(value): + raise bad_request(f"{what} must be a UUID") + return UUID(value) + + +def _reject_constant(_name: str) -> Any: + raise ValueError("NaN/Infinity are not valid JSON") + + +def _object_without_duplicates(pairs: List[tuple]) -> Dict[str, Any]: + result: Dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate key in JSON object") + result[key] = value + return result + + +def parse_json_object(raw: bytes, allowed: Iterable[str], required: Iterable[str]) -> Dict[str, Any]: + """Parse a JSON object body. Malformed JSON, extra or missing fields -> 400.""" + try: + data = json.loads( + raw.decode("utf-8"), + object_pairs_hook=_object_without_duplicates, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad_request("request body must be valid UTF-8 JSON") from None + if not isinstance(data, dict): + raise bad_request("request body must be a JSON object") + allowed_set = set(allowed) + if set(data) - allowed_set: + raise bad_request("request body contains unexpected fields") + missing = [name for name in required if name not in data] + if missing: + raise bad_request("missing required fields: " + ", ".join(missing)) + return data + + +def valid_text(value: Any, minimum: int, maximum: int) -> bool: + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + return False + try: + value.encode("utf-8") # rejects lone surrogates such as "\\ud800" + except UnicodeEncodeError: + return False + return True + + +def valid_password(value: Any) -> bool: + return valid_text(value, 12, 128) + + +def text_field(data: Dict[str, Any], name: str, max_length: int) -> str: + value = data[name] + if not valid_text(value, 1, max_length) or "\x00" in value: + raise bad_request(f"{name} must contain 1..{max_length} valid nonblank characters") + return value + + +def username_field(data: Dict[str, Any]) -> str: + value = data["username"] + if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None: + raise bad_request("username must match [a-z0-9_]{3,32}") + return value + + +def member_ids_field(data: Dict[str, Any]) -> List[UUID]: + value = data["member_ids"] + if not isinstance(value, list) or not 1 <= len(value) <= 100: + raise bad_request("member_ids must be an array of 1..100 user ids") + members = [parse_uuid(item, "member_ids item") for item in value] + if len(set(members)) != len(members): + raise bad_request("member_ids must not contain duplicates") + return members + + +def _single(params: Dict[str, List[str]], name: str) -> Optional[str]: + values = params.get(name) + if values is None: + return None + if len(values) != 1: + raise bad_request(f"{name} must be given at most once") + return values[0] + + +def limit_param(params: Dict[str, List[str]]) -> int: + raw = _single(params, "limit") + if raw is None: + return DEFAULT_LIMIT + if DIGITS_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT: + raise bad_request(f"limit must be an integer 1..{MAX_LIMIT}") + return int(raw) + + +def cursor_param(params: Dict[str, List[str]]) -> Optional[str]: + raw = _single(params, "cursor") + if raw is None: + return None + if not raw or len(raw) > MAX_CURSOR_LENGTH: + raise bad_request("invalid cursor") + return raw diff --git a/student-tests/grade4.py b/student-tests/grade4.py new file mode 100644 index 00000000..5ff82e60 --- /dev/null +++ b/student-tests/grade4.py @@ -0,0 +1,141 @@ +"""Check grade 4 against the local Compose stand using synthetic data.""" + +import base64 +import hashlib +import subprocess +import sys +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path +from threading import Barrier + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run, timestamp + +LOGIN = "/api/v1/auth/sessions" +ME = "/api/v1/users/me" + + +def compose(*args): + result = subprocess.run( + ["docker", "compose", *args], cwd=ROOT, check=True, timeout=60, + stdout=subprocess.PIPE, text=True, encoding="utf-8", + ) + return result.stdout.strip() + + +def basic(registration, password): + pair = f"{registration['username']}:{password}".encode() + return {"Authorization": "Basic " + base64.b64encode(pair).decode()} + + +def race(client): + count = 6 + body = { + "username": "race_" + uuid.uuid4().hex[:20], + "display_name": "Гонка регистрации", + "password": "Синтетический:" + uuid.uuid4().hex, + } + barrier = Barrier(count, timeout=10) + + def attempt(_): + worker = Client(client.base) + barrier.wait() + return worker.call("POST", "/api/v1/users", (201, 409), body=body)["status"] + + with ThreadPoolExecutor(max_workers=count) as workers: + statuses = sorted(workers.map(attempt, range(count))) + check(statuses == [201] + [409] * (count - 1), "duplicate registration race failed") + print(f"PASS registration race ({count} concurrent attempts): {statuses}") + client.call("POST", LOGIN, 201, schema="Session", headers=basic(body, body["password"])) + print("PASS Basic login with a Unicode password containing a colon") + + +def rollback(client, actor): + # The title is generated here from fixed ASCII and a UUID, with no external input. + title = "rollback_" + uuid.uuid4().hex + client.call("POST", "/api/v1/chats", 400, actor=actor, body={ + "title": title, "member_ids": [actor["id"], str(uuid.uuid4())], + }) + remaining = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-XAt", "-v", "ON_ERROR_STOP=1", "-c", + f"SELECT count(*) FROM chats WHERE title = '{title}'") + check(remaining == "0", "a rejected chat left an orphan row") + print("PASS rollback: HTTP 400 and zero rows for rejected chat in PostgreSQL") + + +def wait_ready(client): + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + try: + if client.call("GET", "/health/ready", (200, 503))["status"] == 200: + return + except Failure: + pass + time.sleep(0.3) + raise Failure("API did not become ready") + + +def outage(client, service, actor): + compose("stop", service) + try: + client.call("GET", "/health/live", schema="Health") + client.call("GET", "/health/ready", 503) + failure = client.call("GET", ME, 503, actor=actor) + check(failure["elapsed"] < 5, "dependency failure exceeded HTTP time budget") + client.call("GET", ME, 401, headers={"X-User-Id": actor["id"]}) + finally: + compose("start", service) + started = time.monotonic() + wait_ready(client) + same = client.call("GET", ME, actor=actor, schema="User")["json"] + check(same == actor["user"], "previous session did not recover") + print(f"PASS {service} outage: live=200 ready=503 Bearer=503 " + f"({failure['elapsed']:.3f}s) anonymous=401; same token after start " + f"({time.monotonic() - started:.3f}s from docker compose start return)") + + +def expiry(client): + actor = client.user("expiry") + session = client.call("POST", LOGIN, 201, schema="Session", + headers=basic(actor["registration"], actor["registration"]["password"]))["json"] + actor["token"] = session["token"] + remaining = (timestamp(session["expires_at"]) - datetime.now(timezone.utc)).total_seconds() + check(0 < remaining <= 2.1, "start API with SESSION_TTL_SECONDS=2 for this test") + client.call("GET", ME, actor=actor, schema="User") + time.sleep(remaining + 0.15) + client.call("GET", ME, 401, actor=actor) + print("PASS 2s expiry: same token 200 before deadline and 401 after deadline") + + +def main(): + args = arguments("Lab 2 grade 4 checks (stop and restore dependencies)", + lambda p: p.add_argument("--expiry-only", action="store_true")) + client = Client(args.base_url, args.ca_file, timeout=6) + wait_ready(client) + if args.expiry_only: + expiry(client) + return + race(client) + actor = client.user("outage") + rollback(client, actor) + client.call("POST", LOGIN, 401, headers=basic(actor["registration"], "Wrong:password-123")) + revoked = client.user("logout") + client.call("DELETE", LOGIN + "/current", 204, actor=revoked) + client.call("GET", ME, 401, actor=revoked) + print("PASS wrong password=401, logout=204, revoked token=401") + key = "lab2:session:" + hashlib.sha256(actor["token"].encode()).hexdigest() + ttl = int(compose("exec", "-T", "redis", "redis-cli", "TTL", key)) + check(ttl >= 300, "start the default-TTL stand for the dependency tests") + print(f"PASS Redis session key has TTL ({ttl}s left)") + outage(client, "redis", actor) + client.call("GET", ME, 401, actor=revoked) + print("PASS Redis restart did not revive revoked session") + outage(client, "postgres", actor) + + +if __name__ == "__main__": + run(main) -- 2.54.0