ЛР 2 — БВТ2304 — Риц — на 4 #153
+3
-9
@@ -1,11 +1,5 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Local stand. Replace the empty values with independently generated secrets.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
POSTGRES_PASSWORD=
|
||||
CURSOR_SECRET=
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
scripts/restart.sh text eol=lf
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM python:3.12.12-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app/src \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
# Unprivileged user with a fixed UID; the application never runs as root.
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt /app/
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Only the source code goes into the image (no .env, tests, evidence, .git).
|
||||
# Files stay owned by root, so the app user cannot modify its own code.
|
||||
COPY src/ /app/src/
|
||||
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
|
||||
# docker stop sends SIGTERM; the app drains and exits well within the 10 s budget.
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["python", "/app/src/healthcheck.py"]
|
||||
|
||||
# Exec form: python is PID 1 and receives signals directly (no shell in between).
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -1,37 +1,121 @@
|
||||
# Отчёт — лабораторная 2
|
||||
# Лабораторная 2. Мессенджер с постоянным хранилищем
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
**Риц Дмитрий, БВТ2304**
|
||||
|
||||
## Запуск из чистого clone
|
||||
Целевая оценка — **4**.
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
## Запуск на Windows
|
||||
|
||||
## Архитектура и решения
|
||||
Для запуска нужны Docker Desktop с Linux Engine и Docker Compose. Тесты запускаются на хосте через Python 3.9+. В контейнерах используются Python 3.12.12, PostgreSQL 17.6 и Redis 7.4.6. Команды ниже выполняются из корня репозитория.
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```powershell
|
||||
if (-not (Test-Path .env)) {
|
||||
python -c 'from pathlib import Path; import secrets; Path(".env").write_text(f"APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nPOSTGRES_PASSWORD={secrets.token_hex(24)}\nCURSOR_SECRET={secrets.token_hex(32)}\n", encoding="ascii")'
|
||||
}
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --wait
|
||||
docker compose ps
|
||||
Invoke-RestMethod http://127.0.0.1:8080/health/ready
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
Сервис `migrate` стартует после PostgreSQL и применяет миграции. Его нормальное состояние после работы — `Exited (0)`. API ждёт успешной миграции и готовности Redis. `APP_PORT` в `.env` задаёт опубликованный порт на `127.0.0.1`. При его изменении передайте свой адрес через `--base-url` тестам. `POSTGRES_PASSWORD` — локальный пароль БД, `CURSOR_SECRET` — ключ HMAC для курсоров, `SESSION_TTL_SECONDS` — срок новых сессий. Оба секрета остаются в игнорируемом `.env`. Повторный запуск использует прежний файл и тома.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
`docker compose down` останавливает стенд с сохранением данных. Для намеренного сброса только этого учебного проекта предусмотрена команда `docker compose down -v`. Она удаляет данные и все сессии.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Архитектура
|
||||
|
||||
## Эксперименты и отказы
|
||||
- Из первой лабы остались HTTP-маршруты, проверки запросов, формат ошибок, проверка участников чата и пагинация. При регистрации появился пароль. Для входа используется Basic, после входа клиент передаёт Bearer-токен. Хеш пароля считается через `hashlib.scrypt` с отдельной 16-байтной солью (`N=32768, r=8, p=3`).
|
||||
- В PostgreSQL лежат пользователи, чаты, участники и сообщения. За уникальность логина отвечает `UNIQUE (username)`. Внешний ключ сообщения проверяет, что отправитель входит в чат. Чат и участники записываются в одной транзакции. Если одного из пользователей нет, откатывается и запись самого чата. API возвращает 201 после `COMMIT`.
|
||||
- В Redis лежат сессии. Ключ вычисляется как SHA-256 случайного 256-битного токена, значение — UUID пользователя. При входе Redis получает срок истечения через `PXAT`, при выходе ключ удаляется. Сессии и их сроки переживают рестарт благодаря AOF, `appendfsync always` и тому `redis_data`. Очистка Redis приведёт к повторному входу.
|
||||
- Параметры SQL передаются отдельно от текста запроса. Пул PostgreSQL использует 1–8 соединений, максимум 16 ожидающих и ожидание до 2 секунд. Таймаут подключения — 2 секунды, SQL — 3 секунды, блокировки — 1,5 секунды. Для Redis ограничены соединения (16), подключение и чтение (по 1 секунде), повторы отключены. В контейнере API задана одна попытка DNS с таймаутом 1 секунду. При потере зависимости API возвращает 503 для пути, который её требует. Для защищённого запроса без Bearer ответ остаётся 401.
|
||||
- История сортируется по `(created_at, id)`. Индекс `messages_history (chat_id, created_at, id)` обслуживает равенство по чату, диапазон после курсора и сортировку. Запрашивается `limit + 1` строка, чтобы узнать о следующей странице. Курсор подписан и привязан к чату, а для списка чатов — к пользователю. Следующее чтение видит подтверждённую запись. При конкурентных вставках запись, чей ключ оказался раньше уже пройденной позиции, может появиться только при новом обходе. Часы влияют на порядок, UUID разрешает совпадения времён.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
## Проверки
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
```powershell
|
||||
python scripts/check_contract.py
|
||||
python tests/smoke.py
|
||||
docker compose run --rm migrate
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
# Снять StartedAt контейнеров до и после теста
|
||||
$labIds = docker compose ps -q api postgres redis
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}}' $labIds
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}}' $labIds
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
python student-tests/grade4.py
|
||||
Get-Content -Raw scripts/explain-history.sql | docker compose exec -T postgres psql -X -U messenger -d messenger
|
||||
```
|
||||
|
||||
На Linux публичные проверки выполняются через `make check`, `make test` и `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. Если при переносе в форк потерялся бит исполнения, выполните `chmod +x scripts/restart.sh` или добавьте файл через `git add --chmod=+x scripts/restart.sh`.
|
||||
|
||||
Для expiry временно пересоздаём только API. После проверки возвращаем обычный TTL. У существующих сессий сохраняется исходный срок.
|
||||
|
||||
```powershell
|
||||
$env:SESSION_TTL_SECONDS = '2'
|
||||
try {
|
||||
docker compose up -d --no-deps --force-recreate --wait api
|
||||
python student-tests/grade4.py --expiry-only
|
||||
} finally {
|
||||
Remove-Item Env:SESSION_TTL_SECONDS
|
||||
docker compose up -d --no-deps --force-recreate --wait api
|
||||
}
|
||||
```
|
||||
|
||||
| Критерий | Как проверяется |
|
||||
| --- | --- |
|
||||
| Регистрация, доступ, чаты, сообщения и пагинация | Публичный `tests/smoke.py` регистрирует пользователей, входит, отправляет сообщения, обходит историю и проверяет отказ постороннему. |
|
||||
| Данные и сессия после полного рестарта | `tests/resilience.py` пишет сообщение, выполняет restart hook, повторяет запросы с прежним токеном. `StartedAt` показывает перезапуск API, PostgreSQL и Redis. |
|
||||
| Повторяемые миграции | Повторный `docker compose run --rm migrate` сверяет реестр версий. |
|
||||
| Гонка и атомарность | Шесть одновременных регистраций одного имени дают один 201 и пять 409. Неизвестный участник даёт 400 и ноль строк с тестовым названием чата в БД. |
|
||||
| Redis, logout, ошибки зависимостей | Тест проверяет TTL ключа, 401 при неверном пароле и после logout. По очереди останавливает Redis и PostgreSQL. Проверяет 503 для readiness и действительной сессии и 401 без Bearer. После запуска проверяет прежний токен. |
|
||||
| Истечение сессии | При TTL 2 секунды один токен даёт сначала 200, после срока — 401. |
|
||||
| План истории | `scripts/explain-history.sql` вставляет 10 000 строк через `generate_series`, обновляет статистику, снимает `EXPLAIN (ANALYZE, BUFFERS)` первой страницы и страницы после 9000-й записи. Синтетические строки откатываются. |
|
||||
|
||||
## Наблюдаемые результаты
|
||||
|
||||
На Compose-стенде под Windows первый прогон миграций применил `001_initial`, второй вернул `Schema is up to date`. Публичный smoke и проверка полного рестарта прошли. После рестарта изменилось время запуска всех трёх контейнеров, а ранее выданный токен прочитал сохранённое сообщение. [Вывод проверок](evidence/compose-checks.txt)
|
||||
|
||||
Шесть одновременных регистраций дали один 201 и пять 409. После отката чата с неизвестным участником в PostgreSQL осталось ноль строк с его названием. Тест подтвердил короткий TTL, отзыв сессии и сохранение отзыва после перезапуска Redis. При остановке любой зависимости readiness и запрос с действующей сессией вернули 503. Запрос с токеном получил 503 за 2,012 с при остановке Redis и за 2,011 с при остановке PostgreSQL. После запуска зависимости прежний токен снова работал. [Вывод проверок](evidence/compose-checks.txt)
|
||||
|
||||
На синтетических 10 000 сообщениях обе страницы получили по 51 строке через `Index Scan using messages_history`. Первая страница затронула три буфера и заняла 0,073 мс. Глубокая страница после 9000-й строки затронула три буфера и заняла 0,185 мс. У глубокой страницы сравнение пары `(created_at, id)` находится в `Index Cond`. Это один локальный прогон, отражающий форму плана и фактическое число строк. [Вывод EXPLAIN](evidence/history-plan.txt)
|
||||
|
||||
## Границы эксперимента
|
||||
|
||||
Стенд рассчитан на локальные синтетические данные. PostgreSQL и Redis представлены по одному экземпляру. Тесты проверяют обычный рестарт с сохранёнными томами и временный отказ зависимостей. Потеря тома, отказ диска и восстановление на другом хосте в этот сценарий не входят.
|
||||
|
||||
## Порядок демонстрации
|
||||
|
||||
1. Открыть терминал в папке проекта, запустить Compose и показать `docker compose ps` и `/health/ready`.
|
||||
2. Войти с Basic и отправить сообщение с Bearer по командам ниже.
|
||||
3. Запустить тест перезапуска и сравнить `StartedAt` трёх контейнеров. Тест прочитает сообщение с прежним токеном.
|
||||
4. Показать план страницы после 9000-й записи. Условие `ROW(created_at, id) > ...` стоит в `Index Cond`, поэтому PostgreSQL начинает чтение сразу после курсора.
|
||||
5. Запустить `grade4.py`. Сравнить ответы 401 без действующей сессии и 503 при отказе зависимости.
|
||||
|
||||
У PostgreSQL включены `synchronous_commit` и `fsync`. При этих настройках сервер подтверждает `COMMIT` после записи WAL на диск. Redis восстанавливает сессии из AOF.
|
||||
|
||||
### Ручной вход и сообщение в PowerShell
|
||||
|
||||
```powershell
|
||||
$labApi = 'http://127.0.0.1:8080/api/v1'
|
||||
$labName = 'demo_' + [guid]::NewGuid().ToString('N').Substring(0, 12)
|
||||
$labPass = 'Demo:' + [guid]::NewGuid().ToString('N')
|
||||
$labRegistration = @{username=$labName; display_name='Демо'; password=$labPass} | ConvertTo-Json
|
||||
$labUser = Invoke-RestMethod -Method Post -Uri "$labApi/users" -ContentType 'application/json; charset=utf-8' -Body $labRegistration
|
||||
$labBasic = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes("${labName}:$labPass"))
|
||||
$labSession = Invoke-RestMethod -Method Post -Uri "$labApi/auth/sessions" -Headers @{Authorization="Basic $labBasic"}
|
||||
$labAuth = @{Authorization="Bearer $($labSession.token)"}
|
||||
Invoke-RestMethod -Uri "$labApi/users/me" -Headers $labAuth
|
||||
$labChatJson = @{title='Чат для защиты'; member_ids=@($labUser.id)} | ConvertTo-Json
|
||||
$labChat = Invoke-RestMethod -Method Post -Uri "$labApi/chats" -Headers $labAuth -ContentType 'application/json; charset=utf-8' -Body $labChatJson
|
||||
$labHistory = "$labApi/chats/$($labChat.id)/messages"
|
||||
$labMessage = @{text='Привет, сообщение сохранено!'} | ConvertTo-Json
|
||||
Invoke-RestMethod -Method Post -Uri $labHistory -Headers $labAuth -ContentType 'application/json; charset=utf-8' -Body $labMessage
|
||||
Invoke-RestMethod -Uri $labHistory -Headers $labAuth | ConvertTo-Json -Depth 4
|
||||
```
|
||||
|
||||
## Источники и помощь ИИ
|
||||
|
||||
Основа работы — каркас курса и код первой лабы. Для реализации, тестов и отчёта использовался Codex.
|
||||
|
||||
Документация по трём компонентам. [Индексы PostgreSQL](https://www.postgresql.org/docs/17/indexes-ordering.html), [пул Psycopg](https://www.psycopg.org/psycopg3/docs/advanced/pool.html), [AOF в Redis](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/).
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Generate .env as described in REPORT.md}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
redis:
|
||||
image: redis:7.4.6-bookworm
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", ""]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
migrate:
|
||||
image: labkos-two-api:lab2
|
||||
build: .
|
||||
environment:
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Generate .env as described in REPORT.md}
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
api:
|
||||
image: labkos-two-api:lab2
|
||||
build: .
|
||||
environment:
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Generate .env as described in REPORT.md}
|
||||
CURSOR_SECRET: ${CURSOR_SECRET:?Generate .env as described in REPORT.md}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
dns_opt:
|
||||
- timeout:1
|
||||
- attempts:1
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,43 @@
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
labkos_two-api-1 labkos-two-api:lab2 "python -m messenger" api 9 seconds ago Up 7 seconds (healthy) 127.0.0.1:8080->8080/tcp
|
||||
labkos_two-migrate-1 labkos-two-api:lab2 "python -m messenger…" migrate 2 minutes ago Exited (0) 2 minutes ago
|
||||
labkos_two-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 2 minutes ago Up About a minute (healthy) 5432/tcp
|
||||
labkos_two-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 2 minutes ago Up About a minute (healthy) 6379/tcp
|
||||
Applied: 001_initial
|
||||
Schema is up to date
|
||||
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||
Before full restart (UTC)
|
||||
/labkos_two-api-1 2026-11-05T22:42:13.724276004Z
|
||||
/labkos_two-postgres-1 2026-11-05T22:40:53.482741183Z
|
||||
/labkos_two-redis-1 2026-11-05T22:40:51.310589866Z
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
After full restart (UTC)
|
||||
/labkos_two-api-1 2026-11-05T22:42:39.6528577Z
|
||||
/labkos_two-postgres-1 2026-11-05T22:42:39.378949085Z
|
||||
/labkos_two-redis-1 2026-11-05T22:42:39.28935528Z
|
||||
appendonly
|
||||
yes
|
||||
appendfsync
|
||||
always
|
||||
synchronous_commit
|
||||
--------------------
|
||||
on
|
||||
(1 row)
|
||||
|
||||
fsync
|
||||
-------
|
||||
on
|
||||
(1 row)
|
||||
|
||||
PASS registration race (6 concurrent attempts): [201, 409, 409, 409, 409, 409]
|
||||
PASS Basic login with a Unicode password containing a colon
|
||||
PASS rollback: HTTP 400 and zero rows for rejected chat in PostgreSQL
|
||||
PASS wrong password=401, logout=204, revoked token=401
|
||||
PASS Redis session key has TTL (3597s left)
|
||||
PASS redis outage: live=200 ready=503 Bearer=503 (2.012s) anonymous=401; same token after start (0.060s from docker compose start return)
|
||||
PASS Redis restart did not revive revoked session
|
||||
PASS postgres outage: live=200 ready=503 Bearer=503 (2.011s) anonymous=401; same token after start (0.088s from docker compose start return)
|
||||
PASS 2s expiry: same token 200 before deadline and 401 after deadline
|
||||
PASS manual PowerShell login, chat, message and history from REPORT.md
|
||||
@@ -0,0 +1,38 @@
|
||||
BEGIN
|
||||
INSERT 0 1
|
||||
INSERT 0 1
|
||||
INSERT 0 1
|
||||
INSERT 0 10000
|
||||
ANALYZE
|
||||
messages_in_test_chat
|
||||
-----------------------
|
||||
10000
|
||||
(1 row)
|
||||
|
||||
First page (50 items plus one lookahead)
|
||||
QUERY PLAN
|
||||
-----------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (cost=0.29..4.28 rows=51 width=78) (actual time=0.024..0.044 rows=51 loops=1)
|
||||
Buffers: shared hit=3
|
||||
-> Index Scan using messages_history on messages (cost=0.29..782.85 rows=10000 width=78) (actual time=0.022..0.035 rows=51 loops=1)
|
||||
Index Cond: (chat_id = '<fixture-uuid>'::uuid)
|
||||
Buffers: shared hit=3
|
||||
Planning:
|
||||
Buffers: shared hit=18
|
||||
Planning Time: 0.268 ms
|
||||
Execution Time: 0.073 ms
|
||||
(9 rows)
|
||||
|
||||
Deep page (after message 9000)
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (cost=0.29..15.36 rows=51 width=78) (actual time=0.081..0.103 rows=51 loops=1)
|
||||
Buffers: shared hit=3
|
||||
-> Index Scan using messages_history on messages (cost=0.29..296.39 rows=1002 width=78) (actual time=0.079..0.094 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = '<fixture-uuid>'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 02:30:00+00'::timestamp with time zone, '<fixture-uuid>'::uuid)))
|
||||
Buffers: shared hit=3
|
||||
Planning Time: 0.178 ms
|
||||
Execution Time: 0.185 ms
|
||||
(7 rows)
|
||||
|
||||
ROLLBACK
|
||||
@@ -0,0 +1,3 @@
|
||||
psycopg[binary]==3.2.10
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.4.0
|
||||
@@ -0,0 +1,33 @@
|
||||
\set ON_ERROR_STOP on
|
||||
BEGIN;
|
||||
|
||||
INSERT INTO users (username, display_name, password_salt, password_hash)
|
||||
VALUES ('explain_' || left(replace(gen_random_uuid()::text, '-', ''), 24),
|
||||
'EXPLAIN fixture', decode(repeat('00', 16), 'hex'), decode(repeat('00', 32), 'hex'))
|
||||
RETURNING id AS fixture_user \gset
|
||||
|
||||
INSERT INTO chats (title) VALUES ('EXPLAIN fixture') RETURNING id AS fixture_chat \gset
|
||||
INSERT INTO chat_members (chat_id, user_id, position) VALUES (:'fixture_chat', :'fixture_user', 1);
|
||||
INSERT INTO messages (chat_id, sender_id, text, created_at)
|
||||
SELECT :'fixture_chat', :'fixture_user', 'synthetic message ' || n,
|
||||
TIMESTAMPTZ '2026-01-01 00:00:00+00' + n * INTERVAL '1 second'
|
||||
FROM generate_series(1, 10000) n;
|
||||
ANALYZE messages;
|
||||
SELECT count(*) AS messages_in_test_chat FROM messages WHERE chat_id = :'fixture_chat';
|
||||
|
||||
\echo First page (50 items plus one lookahead)
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = :'fixture_chat'
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
SELECT created_at AS cursor_time, id AS cursor_id FROM messages
|
||||
WHERE chat_id = :'fixture_chat' AND created_at = TIMESTAMPTZ '2026-01-01 02:30:00+00'
|
||||
\gset
|
||||
|
||||
\echo Deep page (after message 9000)
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = :'fixture_chat'
|
||||
AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
docker compose --project-directory "%~dp0.." restart postgres redis api
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec docker compose restart postgres redis api
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Docker HEALTHCHECK probe: exit 0 if /health/ready answers 200, else 1."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
port = os.environ.get("APP_PORT", "8080")
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
|
||||
sys.exit(0 if response.status == 200 else 1)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,3 @@
|
||||
"""PostgreSQL and Redis messenger for MTUSI Lab 2."""
|
||||
|
||||
__version__ = "2.0.0"
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Run the Lab 2 HTTP API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from . import __version__
|
||||
from .config import Config
|
||||
from .http_api import MessengerServer
|
||||
from .limits import Limits
|
||||
from .logs import log
|
||||
from .sessions import Sessions
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main() -> None:
|
||||
config = Config.from_env()
|
||||
limits = Limits()
|
||||
store = Store(config.cursor_secret)
|
||||
sessions = Sessions(config.session_ttl)
|
||||
server = MessengerServer((config.host, config.port), store, sessions, limits, config.max_body_bytes)
|
||||
stop_requested = threading.Event()
|
||||
|
||||
def on_signal(signum: int, _frame) -> None:
|
||||
if stop_requested.is_set():
|
||||
return
|
||||
stop_requested.set()
|
||||
server.draining = True # readiness -> 503, new API requests -> 503
|
||||
log("info", "shutdown_started", signal=signal.Signals(signum).name)
|
||||
# shutdown() waits for serve_forever() to return, so it must run in another thread.
|
||||
threading.Thread(target=server.shutdown, name="shutdown", daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, on_signal)
|
||||
signal.signal(signal.SIGINT, on_signal)
|
||||
|
||||
log("info", "started", version=__version__, host=config.host, port=config.port, env=config.env,
|
||||
pid=os.getpid(), storage="postgres", session_store="redis", max_body_bytes=config.max_body_bytes,
|
||||
**limits.describe())
|
||||
limits.start_sampler()
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
|
||||
stopped_at = time.monotonic()
|
||||
server.server_close() # stop accepting connections: the listening socket is closed
|
||||
finished = server.wait_idle(timeout=config.shutdown_timeout)
|
||||
store.pool.close()
|
||||
sessions.redis.close()
|
||||
log("info" if finished else "warning", "shutdown_complete",
|
||||
in_flight_finished=finished,
|
||||
drain_ms=round((time.monotonic() - stopped_at) * 1000, 1))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Local configuration from environment variables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .logs import log
|
||||
|
||||
|
||||
def postgres_options() -> dict:
|
||||
return {
|
||||
"host": os.environ.get("POSTGRES_HOST", "postgres"),
|
||||
"port": env_int("POSTGRES_PORT", 5432, 1, 65535),
|
||||
"dbname": os.environ.get("POSTGRES_DB", "messenger"),
|
||||
"user": os.environ.get("POSTGRES_USER", "messenger"),
|
||||
"password": os.environ.get("POSTGRES_PASSWORD"),
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=3000 -c lock_timeout=1500 -c timezone=UTC",
|
||||
}
|
||||
|
||||
|
||||
def env_int(name: str, default: int, low: int, high: int) -> int:
|
||||
raw = os.environ.get(name, "").strip() or str(default)
|
||||
if not raw.isdigit() or not low <= int(raw) <= high:
|
||||
log("error", "bad_config", variable=name, allowed=f"{low}..{high}")
|
||||
sys.exit(2)
|
||||
return int(raw)
|
||||
|
||||
|
||||
def env_bool(name: str, default: bool) -> bool:
|
||||
raw = os.environ.get(name, "").strip().lower()
|
||||
if not raw:
|
||||
return default
|
||||
if raw in ("1", "true", "yes", "on"):
|
||||
return True
|
||||
if raw in ("0", "false", "no", "off"):
|
||||
return False
|
||||
log("error", "bad_config", variable=name, allowed="0/1")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
host: str
|
||||
port: int
|
||||
env: str
|
||||
shutdown_timeout: int
|
||||
max_body_bytes: int
|
||||
session_ttl: int
|
||||
cursor_secret: bytes
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "Config":
|
||||
secret = os.environ.get("CURSOR_SECRET", "").encode()
|
||||
if len(secret) < 32:
|
||||
log("error", "bad_config", variable="CURSOR_SECRET", minimum_length=32)
|
||||
sys.exit(2)
|
||||
return cls(
|
||||
host=os.environ.get("APP_HOST", "").strip() or "0.0.0.0",
|
||||
port=env_int("APP_PORT", 8080, 1, 65535),
|
||||
env=os.environ.get("APP_ENV", "lab"),
|
||||
shutdown_timeout=env_int("SHUTDOWN_TIMEOUT_SECONDS", 8, 1, 9),
|
||||
max_body_bytes=env_int("MAX_BODY_BYTES", 64 * 1024, 1024, 16 * 1024 * 1024),
|
||||
session_ttl=env_int("SESSION_TTL_SECONDS", 3600, 1, 31_536_000),
|
||||
cursor_secret=secret,
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
"""API errors in the common contract format: {"error": {"code", "message", "request_id"}}."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""An error that is turned into a JSON error response."""
|
||||
|
||||
def __init__(self, status: int, code: str, message: str, close: bool = False,
|
||||
retry_after: Optional[int] = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
# close=True: the request body may not have been read, so the connection
|
||||
# must not be reused for another request.
|
||||
self.close = close
|
||||
self.retry_after = retry_after # seconds, sent as the Retry-After header
|
||||
|
||||
|
||||
def bad_request(message: str, close: bool = False) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message, close)
|
||||
|
||||
|
||||
def unauthorized() -> ApiError:
|
||||
return ApiError(401, "unauthorized", "invalid credentials or session")
|
||||
|
||||
|
||||
def not_found(message: str = "resource not found") -> ApiError:
|
||||
return ApiError(404, "not_found", message)
|
||||
|
||||
|
||||
def conflict(message: str) -> ApiError:
|
||||
return ApiError(409, "conflict", message)
|
||||
|
||||
|
||||
def payload_too_large(message: str) -> ApiError:
|
||||
return ApiError(413, "payload_too_large", message, close=True)
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, "unavailable", message)
|
||||
|
||||
|
||||
def internal_error() -> ApiError:
|
||||
# Never expose stack traces or internal details to the client.
|
||||
return ApiError(500, "internal_error", "internal server error")
|
||||
|
||||
|
||||
_CODE_BY_STATUS = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
}
|
||||
|
||||
|
||||
def code_for_status(status: int) -> str:
|
||||
"""Contract error code for an arbitrary HTTP status (used for low-level HTTP errors)."""
|
||||
if status in _CODE_BY_STATUS:
|
||||
return _CODE_BY_STATUS[status]
|
||||
return "internal_error" if status >= 500 else "invalid_request"
|
||||
@@ -0,0 +1,432 @@
|
||||
"""HTTP layer: routing, session authentication, JSON responses, request logs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
|
||||
from . import validation as v
|
||||
from .errors import ApiError, bad_request, code_for_status, internal_error, not_found, unauthorized, unavailable
|
||||
from .limits import Limits
|
||||
from .logs import log
|
||||
from .sessions import Sessions, credentials, password_digest, verify_password
|
||||
from .store import Store
|
||||
|
||||
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
class MessengerServer(ThreadingHTTPServer):
|
||||
"""Thread per connection with bounded admission and graceful shutdown."""
|
||||
|
||||
daemon_threads = True # idle keep-alive connections must not block process exit
|
||||
request_queue_size = 128
|
||||
|
||||
def __init__(self, address: Tuple[str, int], store: Store, sessions: Sessions,
|
||||
limits: Limits, max_body_bytes: int):
|
||||
self.store = store
|
||||
self.sessions = sessions
|
||||
self.limits = limits
|
||||
self.max_body_bytes = max_body_bytes
|
||||
self.draining = False
|
||||
self._in_flight = 0
|
||||
self._connections = 0
|
||||
self._idle = threading.Condition()
|
||||
super().__init__(address, Handler)
|
||||
|
||||
def server_bind(self) -> None:
|
||||
# HTTPServer.server_bind() does a reverse DNS lookup (getfqdn) that can
|
||||
# hang for seconds inside containers; it is not needed for an API.
|
||||
socketserver.TCPServer.server_bind(self)
|
||||
self.server_name, self.server_port = self.server_address[:2]
|
||||
|
||||
# -- connections: one thread each; refuse new ones above MAX_CONNECTIONS --
|
||||
|
||||
def process_request(self, request, client_address) -> None:
|
||||
with self._idle:
|
||||
refuse = self.limits.enabled and self._connections >= self.limits.max_connections
|
||||
if not refuse:
|
||||
self._connections += 1
|
||||
if refuse:
|
||||
_refuse_connection(request)
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
try:
|
||||
super().process_request(request, client_address)
|
||||
except Exception:
|
||||
with self._idle:
|
||||
self._connections -= 1
|
||||
raise
|
||||
|
||||
def process_request_thread(self, request, client_address) -> None:
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
with self._idle:
|
||||
self._connections -= 1
|
||||
|
||||
# -- requests in progress --
|
||||
|
||||
def request_started(self) -> int:
|
||||
with self._idle:
|
||||
self._in_flight += 1
|
||||
return self._in_flight
|
||||
|
||||
def request_finished(self) -> None:
|
||||
with self._idle:
|
||||
self._in_flight -= 1
|
||||
if self._in_flight == 0:
|
||||
self._idle.notify_all()
|
||||
|
||||
def wait_idle(self, timeout: float) -> bool:
|
||||
"""Wait until no request is being processed; False on timeout."""
|
||||
with self._idle:
|
||||
return self._idle.wait_for(lambda: self._in_flight == 0, timeout=max(timeout, 0))
|
||||
|
||||
|
||||
# ----- route handlers --------------------------------------------------------
|
||||
# Each handler gets (request, user, path params, query) and returns (status, body).
|
||||
|
||||
Result = Tuple[int, Optional[Dict[str, Any]]]
|
||||
|
||||
|
||||
def health_live(req: "Handler", user, params, query) -> Result:
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
|
||||
def _refuse_connection(sock) -> None:
|
||||
"""Answer 503 without starting a thread (connection limit). Runs in the accept loop."""
|
||||
request_id = uuid.uuid4().hex
|
||||
body = json.dumps({"error": {"code": "unavailable", "message": "too many connections, retry later",
|
||||
"request_id": request_id}}).encode()
|
||||
head = (
|
||||
"HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
|
||||
f"Content-Length: {len(body)}\r\nX-Request-Id: {request_id}\r\nRetry-After: 1\r\n"
|
||||
"Connection: close\r\n\r\n"
|
||||
).encode()
|
||||
try:
|
||||
sock.settimeout(0.05)
|
||||
try:
|
||||
sock.recv(65536) # read what already arrived so close() does not reset the connection
|
||||
except OSError:
|
||||
pass
|
||||
sock.sendall(head + body)
|
||||
except OSError:
|
||||
pass
|
||||
log("info", "request", request_id=request_id, method="-", route="-", status=503,
|
||||
duration_ms=0.0, rejected="connections")
|
||||
|
||||
|
||||
def health_ready(req: "Handler", user, params, query) -> Result:
|
||||
if req.server.draining:
|
||||
raise unavailable("server is shutting down")
|
||||
if not req.server.limits.ready():
|
||||
raise unavailable("memory limit reached")
|
||||
req.server.store.ready()
|
||||
req.server.sessions.ready()
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
|
||||
def create_user(req: "Handler", user, params, query) -> Result:
|
||||
fields = ("username", "display_name", "password")
|
||||
data = req.json_body(allowed=fields, required=fields)
|
||||
username = v.username_field(data)
|
||||
display_name = v.text_field(data, "display_name", 100)
|
||||
password = data["password"]
|
||||
if not v.valid_password(password):
|
||||
raise bad_request("password must contain 12..128 valid nonblank characters")
|
||||
salt = secrets.token_bytes(16)
|
||||
return 201, req.server.store.create_user(username, display_name, salt, password_digest(password, salt))
|
||||
|
||||
|
||||
def create_session(req: "Handler", user, params, query) -> Result:
|
||||
if req._read_body():
|
||||
raise bad_request("login does not accept a body")
|
||||
username, password = credentials(req.authorization("basic"))
|
||||
user = req.server.store.credentials(username)
|
||||
if not verify_password(password, user):
|
||||
raise unauthorized()
|
||||
del user["password_hash"], user["password_salt"]
|
||||
token, expires_at = req.server.sessions.create(user["id"])
|
||||
return 201, {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}
|
||||
|
||||
|
||||
def revoke_session(req: "Handler", user, params, query) -> Result:
|
||||
req.server.sessions.revoke(req.authorization("bearer"))
|
||||
return 204, None
|
||||
|
||||
|
||||
def get_me(req: "Handler", user, params, query) -> Result:
|
||||
profile = req.server.store.get_user(user)
|
||||
if profile is None:
|
||||
raise unauthorized()
|
||||
return 200, profile
|
||||
|
||||
|
||||
def create_chat(req: "Handler", user, params, query) -> Result:
|
||||
req.server.limits.check_rate(user) # 429 if this user writes too fast
|
||||
data = req.json_body(allowed=("title", "member_ids"), required=("title", "member_ids"))
|
||||
title = v.text_field(data, "title", 100)
|
||||
members = v.member_ids_field(data)
|
||||
return 201, req.server.store.create_chat(user, title, members)
|
||||
|
||||
|
||||
def list_chats(req: "Handler", user, params, query) -> Result:
|
||||
limit, cursor = v.limit_param(query), v.cursor_param(query)
|
||||
return 200, req.server.store.list_chats(user, limit, cursor)
|
||||
|
||||
|
||||
def get_chat(req: "Handler", user, params, query) -> Result:
|
||||
return 200, req.server.store.get_chat(user, params["chat_id"])
|
||||
|
||||
|
||||
def post_message(req: "Handler", user, params, query) -> Result:
|
||||
req.server.limits.check_rate(user) # 429 if this user writes too fast
|
||||
# sender_id is not accepted in the body: the sender is always the current user.
|
||||
data = req.json_body(allowed=("text",), required=("text",))
|
||||
text = v.text_field(data, "text", 4000)
|
||||
return 201, req.server.store.add_message(user, params["chat_id"], text)
|
||||
|
||||
|
||||
def list_messages(req: "Handler", user, params, query) -> Result:
|
||||
limit, cursor = v.limit_param(query), v.cursor_param(query)
|
||||
return 200, req.server.store.list_messages(user, params["chat_id"], limit, cursor)
|
||||
|
||||
|
||||
def json_value(value):
|
||||
if isinstance(value, uuid.UUID):
|
||||
return str(value)
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
raise TypeError(f"Cannot serialize {type(value).__name__}")
|
||||
|
||||
|
||||
Route = Tuple[str, "re.Pattern[str]", str, Callable[..., Result], bool]
|
||||
|
||||
_CHAT = r"(?P<chat_id>[^/]+)"
|
||||
ROUTES: List[Route] = [
|
||||
# method, path regex, template for logs, handler, requires Bearer
|
||||
("GET", re.compile(r"/health/live"), "/health/live", health_live, False),
|
||||
("GET", re.compile(r"/health/ready"), "/health/ready", health_ready, False),
|
||||
("POST", re.compile(r"/api/v1/users"), "/api/v1/users", create_user, False),
|
||||
("POST", re.compile(r"/api/v1/auth/sessions"), "/api/v1/auth/sessions", create_session, False),
|
||||
("DELETE", re.compile(r"/api/v1/auth/sessions/current"), "/api/v1/auth/sessions/current", revoke_session, False),
|
||||
("GET", re.compile(r"/api/v1/users/me"), "/api/v1/users/me", get_me, True),
|
||||
("POST", re.compile(r"/api/v1/chats"), "/api/v1/chats", create_chat, True),
|
||||
("GET", re.compile(r"/api/v1/chats"), "/api/v1/chats", list_chats, True),
|
||||
("GET", re.compile(r"/api/v1/chats/" + _CHAT), "/api/v1/chats/{chat_id}", get_chat, True),
|
||||
("POST", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", post_message, True),
|
||||
("GET", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", list_messages, True),
|
||||
]
|
||||
|
||||
|
||||
def resolve(method: str, path: str) -> Tuple[Route, Dict[str, str]]:
|
||||
path_matched = False
|
||||
for route in ROUTES:
|
||||
match = route[1].fullmatch(path)
|
||||
if match is None:
|
||||
continue
|
||||
path_matched = True
|
||||
if route[0] == method or (method == "HEAD" and route[0] == "GET"):
|
||||
return route, match.groupdict()
|
||||
if path_matched:
|
||||
raise ApiError(405, "invalid_request", "method not allowed")
|
||||
raise not_found("route not found")
|
||||
|
||||
|
||||
# ----- HTTP handler ----------------------------------------------------------
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server: MessengerServer
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "mtusi-messenger"
|
||||
sys_version = ""
|
||||
timeout = 30 # idle keep-alive connections are closed after this many seconds
|
||||
# TCP_NODELAY: headers and body are written separately; with Nagle's algorithm plus the
|
||||
# client's delayed ACK every keep-alive response waited ~40 ms (found in the track 2 runs).
|
||||
disable_nagle_algorithm = True
|
||||
|
||||
request_id = ""
|
||||
route_template = "-"
|
||||
started = 0.0
|
||||
_body: Optional[bytes] = None
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_PATCH(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_DELETE(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_HEAD(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_OPTIONS(self) -> None:
|
||||
self._handle()
|
||||
|
||||
# -- request processing --
|
||||
|
||||
def _handle(self) -> None:
|
||||
self.started = time.monotonic()
|
||||
incoming = self.headers.get("X-Request-Id", "")
|
||||
self.request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
|
||||
self.route_template = "-"
|
||||
self._body: Optional[bytes] = None
|
||||
self.server.request_started()
|
||||
slot = False
|
||||
try:
|
||||
if self.command in ("POST", "PUT", "PATCH", "DELETE"):
|
||||
# Read the body first: an early error then never leaves unread bytes
|
||||
# on the connection, so keep-alive works even for 429/503 answers.
|
||||
self._read_body()
|
||||
split = urlsplit(self.path)
|
||||
route, params = resolve(self.command, split.path)
|
||||
self.route_template = route[2]
|
||||
is_health = route[2] in ("/health/live", "/health/ready")
|
||||
if self.server.draining and not is_health:
|
||||
raise unavailable("server is shutting down")
|
||||
if not is_health:
|
||||
slot = self.server.limits.acquire_slot() # bounded queue; 503 when overloaded
|
||||
user = self.server.sessions.identify(self.authorization("bearer")) if route[4] else None
|
||||
if "chat_id" in params:
|
||||
params["chat_id"] = v.parse_uuid(unquote(params["chat_id"]), "chat_id")
|
||||
query = parse_qs(split.query, keep_blank_values=True)
|
||||
status, payload = route[3](self, user, params, query)
|
||||
self._send_json(status, payload)
|
||||
except ApiError as error:
|
||||
if error.close:
|
||||
self.close_connection = True
|
||||
self._safe_send_error(error)
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError):
|
||||
self.close_connection = True # the client went away; nothing to answer
|
||||
except Exception as error:
|
||||
log("error", "unhandled_exception", request_id=self.request_id,
|
||||
route=self.route_template, exception_type=type(error).__name__)
|
||||
self.close_connection = True
|
||||
self._safe_send_error(internal_error())
|
||||
finally:
|
||||
if slot:
|
||||
self.server.limits.release_slot()
|
||||
self.server.request_finished()
|
||||
|
||||
def _safe_send_error(self, error: ApiError) -> None:
|
||||
try:
|
||||
self._send_error_json(error)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
self.close_connection = True
|
||||
|
||||
def authorization(self, scheme: str) -> str:
|
||||
values = self.headers.get_all("Authorization") or []
|
||||
if len(values) != 1:
|
||||
raise unauthorized()
|
||||
parts = values[0].split()
|
||||
if len(parts) != 2 or parts[0].lower() != scheme:
|
||||
raise unauthorized()
|
||||
return parts[1]
|
||||
|
||||
def _read_body(self) -> bytes:
|
||||
if self._body is not None:
|
||||
return self._body
|
||||
if self.headers.get("Transfer-Encoding"):
|
||||
raise bad_request("chunked request bodies are not supported", close=True)
|
||||
lengths = self.headers.get_all("Content-Length") or []
|
||||
if not lengths:
|
||||
self._body = b""
|
||||
return self._body
|
||||
if len(lengths) != 1 or not re.fullmatch(r"[0-9]{1,16}", lengths[0]):
|
||||
raise bad_request("invalid Content-Length", close=True)
|
||||
length = int(lengths[0])
|
||||
if length > self.server.max_body_bytes:
|
||||
raise ApiError(413, "payload_too_large", "request body is too large", close=True)
|
||||
self._body = self.rfile.read(length)
|
||||
if len(self._body) != length:
|
||||
raise bad_request("incomplete request body", close=True)
|
||||
return self._body
|
||||
|
||||
def json_body(self, allowed, required) -> Dict[str, Any]:
|
||||
raw = self._read_body()
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise bad_request("Content-Type must be application/json")
|
||||
charset = self.headers.get_content_charset()
|
||||
if charset not in (None, "utf-8"):
|
||||
raise bad_request("request body must be UTF-8")
|
||||
return v.parse_json_object(raw, allowed, required)
|
||||
|
||||
# -- responses --
|
||||
|
||||
def _send_json(self, status: int, payload: Optional[Dict[str, Any]], retry_after: Optional[int] = None) -> None:
|
||||
body = b"" if payload is None else json.dumps(
|
||||
payload, ensure_ascii=False, separators=(",", ":"), default=json_value
|
||||
).encode("utf-8")
|
||||
if self.command in ("POST", "PUT", "PATCH", "DELETE") and self._body is None:
|
||||
# The body was never read (e.g. error before parsing): do not reuse the connection.
|
||||
self.close_connection = True
|
||||
if self.server.draining:
|
||||
self.close_connection = True
|
||||
self.send_response(status)
|
||||
if status != 204:
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", self.request_id)
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
if retry_after is not None:
|
||||
self.send_header("Retry-After", str(retry_after))
|
||||
if status == 401:
|
||||
challenge = 'Bearer realm="messenger"'
|
||||
if self.route_template == "/api/v1/auth/sessions":
|
||||
challenge = 'Basic realm="messenger", charset="UTF-8"'
|
||||
self.send_header("WWW-Authenticate", challenge)
|
||||
if status == 405:
|
||||
path = urlsplit(getattr(self, "path", "") or "").path
|
||||
allow = sorted({r[0] for r in ROUTES if r[1].fullmatch(path)})
|
||||
if allow:
|
||||
self.send_header("Allow", ", ".join(allow))
|
||||
if self.close_connection:
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
log("info", "request", request_id=self.request_id, method=self.command,
|
||||
route=self.route_template, status=status,
|
||||
duration_ms=round((time.monotonic() - self.started) * 1000, 2))
|
||||
|
||||
def _send_error_json(self, error: ApiError) -> None:
|
||||
self._send_json(error.status, {
|
||||
"error": {"code": error.code, "message": error.message, "request_id": self.request_id}
|
||||
}, retry_after=error.retry_after)
|
||||
|
||||
def send_error(self, code: int, message: Optional[str] = None, explain: Optional[str] = None) -> None:
|
||||
"""Low-level HTTP errors (bad request line, unsupported method) in the same JSON format."""
|
||||
self.request_id = uuid.uuid4().hex
|
||||
self.started = time.monotonic()
|
||||
self.route_template = "-"
|
||||
self._body = None
|
||||
self.close_connection = True
|
||||
if getattr(self, "command", None) is None:
|
||||
self.command = "GET"
|
||||
status = 405 if code == 501 else code # 501 = method unknown to http.server
|
||||
self._safe_send_error(ApiError(status, code_for_status(status), "invalid HTTP request"))
|
||||
|
||||
def version_string(self) -> str:
|
||||
return self.server_version
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None: # noqa: A002
|
||||
pass # replaced by structured logs in _send_json
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Bound HTTP connections, concurrent work and writes per user."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from .config import env_bool, env_int
|
||||
from .errors import ApiError
|
||||
|
||||
try:
|
||||
_PAGE_SIZE = os.sysconf("SC_PAGE_SIZE")
|
||||
except (AttributeError, ValueError, OSError):
|
||||
_PAGE_SIZE = 4096
|
||||
|
||||
|
||||
def read_rss() -> Optional[int]:
|
||||
"""Resident memory of this process in bytes (Linux /proc), None elsewhere."""
|
||||
try:
|
||||
with open("/proc/self/statm") as f:
|
||||
return int(f.read().split()[1]) * _PAGE_SIZE
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _read_limit(path: str) -> Optional[int]:
|
||||
try:
|
||||
with open(path) as f:
|
||||
raw = f.read().strip()
|
||||
except OSError:
|
||||
return None
|
||||
if not raw.isdigit():
|
||||
return None # cgroup v2 says "max" when there is no limit
|
||||
value = int(raw)
|
||||
return value if value < (1 << 50) else None # cgroup v1 uses a huge number for "no limit"
|
||||
|
||||
|
||||
def detect_memory_limit() -> Optional[int]:
|
||||
"""Memory limit of the container (cgroup v2 or v1), None if unlimited/unknown."""
|
||||
candidates: List[str] = ["/sys/fs/cgroup/memory.max"]
|
||||
try:
|
||||
with open("/proc/self/cgroup") as f:
|
||||
for line in f:
|
||||
_, controllers, path = line.rstrip("\n").split(":", 2)
|
||||
if controllers == "":
|
||||
candidates.append(f"/sys/fs/cgroup{path}/memory.max")
|
||||
elif "memory" in controllers.split(","):
|
||||
candidates.append(f"/sys/fs/cgroup/memory{path}/memory.limit_in_bytes")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
candidates.append("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
||||
for path in candidates:
|
||||
value = _read_limit(path)
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
class Limits:
|
||||
def __init__(self) -> None:
|
||||
self.enabled = env_bool("LIMITS_ENABLED", True)
|
||||
# Bounded request queue: MAX_IN_FLIGHT requests are processed at once, up to
|
||||
# MAX_QUEUE more wait for a slot at most QUEUE_TIMEOUT_MS; the rest get 503 at once.
|
||||
self.max_in_flight = env_int("MAX_IN_FLIGHT", 8, 1, 10_000)
|
||||
self.max_queue = env_int("MAX_QUEUE", 64, 0, 100_000)
|
||||
self.queue_timeout = env_int("QUEUE_TIMEOUT_MS", 1000, 1, 60_000) / 1000
|
||||
self._slots = threading.BoundedSemaphore(self.max_in_flight)
|
||||
self.waiting = 0
|
||||
self.max_connections = env_int("MAX_CONNECTIONS", 256, 1, 100_000)
|
||||
self.user_write_rate = env_int("USER_WRITE_RATE", 50, 1, 100_000) # tokens per second
|
||||
self.user_write_burst = env_int("USER_WRITE_BURST", 100, 1, 100_000)
|
||||
configured = env_int("MEMORY_LIMIT_BYTES", 0, 0, 1 << 50)
|
||||
self.memory_limit = configured or detect_memory_limit()
|
||||
self.hard_percent = env_int("MEMORY_HARD_PERCENT", 90, 10, 99)
|
||||
self.rss = read_rss() or 0
|
||||
self._lock = threading.Lock()
|
||||
self._buckets: Dict[UUID, List[float]] = {}
|
||||
|
||||
# ----- description / stats ------------------------------------------------
|
||||
|
||||
def describe(self) -> Dict[str, object]:
|
||||
return {
|
||||
"limits_enabled": self.enabled,
|
||||
"max_in_flight": self.max_in_flight,
|
||||
"max_queue": self.max_queue,
|
||||
"queue_timeout_ms": int(self.queue_timeout * 1000),
|
||||
"max_connections": self.max_connections,
|
||||
"user_write_rate": self.user_write_rate,
|
||||
"user_write_burst": self.user_write_burst,
|
||||
"memory_limit_bytes": self.memory_limit,
|
||||
"memory_hard_percent": self.hard_percent,
|
||||
}
|
||||
|
||||
def _over(self, percent: int) -> bool:
|
||||
return bool(self.enabled and self.memory_limit and self.rss >= self.memory_limit * percent / 100)
|
||||
|
||||
# ----- checks (each raises ApiError when the limit is exceeded) ------------
|
||||
|
||||
def acquire_slot(self) -> bool:
|
||||
"""Admission for every non-health request, before any work is done.
|
||||
|
||||
Returns True when a processing slot was taken (the caller must call
|
||||
release_slot()), False when limits are off. Raises 503 when overloaded.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
if self._over(self.hard_percent):
|
||||
raise ApiError(503, "unavailable", "memory limit reached, retry later", retry_after=5)
|
||||
if self._slots.acquire(blocking=False):
|
||||
return True
|
||||
with self._lock:
|
||||
queue_full = self.waiting >= self.max_queue
|
||||
if not queue_full:
|
||||
self.waiting += 1
|
||||
if queue_full:
|
||||
raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1)
|
||||
try:
|
||||
acquired = self._slots.acquire(timeout=self.queue_timeout)
|
||||
finally:
|
||||
with self._lock:
|
||||
self.waiting -= 1
|
||||
if not acquired:
|
||||
raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1)
|
||||
return True
|
||||
|
||||
def release_slot(self) -> None:
|
||||
self._slots.release()
|
||||
|
||||
def ready(self) -> bool:
|
||||
return not self._over(self.hard_percent)
|
||||
|
||||
def check_rate(self, user_id: UUID) -> None:
|
||||
"""Token bucket per user for write requests."""
|
||||
if not self.enabled:
|
||||
return
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
bucket = self._buckets.get(user_id)
|
||||
if bucket is None:
|
||||
bucket = self._buckets[user_id] = [float(self.user_write_burst), now]
|
||||
tokens = min(self.user_write_burst, bucket[0] + (now - bucket[1]) * self.user_write_rate)
|
||||
bucket[1] = now
|
||||
if tokens >= 1:
|
||||
bucket[0] = tokens - 1
|
||||
return
|
||||
bucket[0] = tokens
|
||||
wait = max(1, math.ceil((1 - tokens) / self.user_write_rate))
|
||||
raise ApiError(429, "rate_limited", "too many writes, slow down", retry_after=wait)
|
||||
|
||||
# ----- background sampler -------------------------------------------------
|
||||
|
||||
def start_sampler(self) -> None:
|
||||
"""Refresh RSS for request admission and readiness."""
|
||||
|
||||
def loop() -> None:
|
||||
while True:
|
||||
rss = read_rss()
|
||||
if rss is not None:
|
||||
self.rss = rss
|
||||
time.sleep(0.1)
|
||||
|
||||
threading.Thread(target=loop, name="limits-sampler", daemon=True).start()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Structured logs: one JSON object per line on stdout.
|
||||
|
||||
Logs never contain message texts, request bodies, query strings or headers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def log(level: str, event: str, **fields: Any) -> None:
|
||||
record = {
|
||||
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
|
||||
"level": level,
|
||||
"event": event,
|
||||
**fields,
|
||||
}
|
||||
line = json.dumps(record, ensure_ascii=False, default=str)
|
||||
with _lock:
|
||||
sys.stdout.write(line + "\n")
|
||||
sys.stdout.flush()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Apply versioned migrations under one PostgreSQL transaction and lock."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import postgres_options
|
||||
|
||||
|
||||
def main() -> None:
|
||||
migrations = Path(__file__).with_name("migrations").glob("*.sql")
|
||||
with psycopg.connect(**postgres_options()) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(202602)")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
""")
|
||||
applied = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
pending = []
|
||||
for path in sorted(migrations):
|
||||
if path.stem not in applied:
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.stem,))
|
||||
pending.append(path.stem)
|
||||
print("Applied: " + ", ".join(pending) if pending else "Schema is up to date")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash bytea NOT NULL,
|
||||
password_salt bytea NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position integer NOT NULL CHECK (position BETWEEN 1 AND 100),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id, chat_id);
|
||||
CREATE INDEX chats_order ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,45 @@
|
||||
"""The lab 1 signed cursor with a stable key and a PostgreSQL sort position."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from .errors import bad_request
|
||||
|
||||
|
||||
class Cursors:
|
||||
def __init__(self, secret: bytes):
|
||||
self.secret = secret
|
||||
|
||||
def _sign(self, payload: bytes) -> bytes:
|
||||
return hmac.new(self.secret, payload, hashlib.sha256).digest()[:16]
|
||||
|
||||
def decode(self, scope: str, cursor: str) -> tuple[datetime, UUID]:
|
||||
try:
|
||||
padded = cursor + "=" * (-len(cursor) % 4)
|
||||
raw = base64.b64decode(padded, altchars=b"-_", validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
raise bad_request("invalid cursor") from None
|
||||
payload, signature = raw[:-16], raw[-16:]
|
||||
if not payload or not hmac.compare_digest(signature, self._sign(payload)):
|
||||
raise bad_request("invalid cursor")
|
||||
cursor_scope, timestamp, object_id = json.loads(payload)
|
||||
if cursor_scope != scope:
|
||||
raise bad_request("cursor belongs to another resource")
|
||||
return datetime.fromisoformat(timestamp), UUID(object_id)
|
||||
|
||||
def page(self, scope: str, items: list[dict], limit: int) -> dict:
|
||||
cursor = None
|
||||
if len(items) > limit:
|
||||
items = items[:limit]
|
||||
last = items[-1]
|
||||
payload = json.dumps(
|
||||
[scope, last["created_at"].isoformat(), str(last["id"])],
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
cursor = base64.urlsafe_b64encode(payload + self._sign(payload)).decode().rstrip("=")
|
||||
return {"items": items, "next_cursor": cursor}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Basic credentials, scrypt password hashes and opaque Redis sessions."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.exceptions import RedisError
|
||||
from redis.retry import Retry
|
||||
|
||||
from .errors import unauthorized, unavailable
|
||||
from .validation import USERNAME_RE, valid_password
|
||||
|
||||
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{43}")
|
||||
|
||||
|
||||
def credentials(header: str) -> tuple[str, str]:
|
||||
try:
|
||||
decoded = base64.b64decode(header, validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError):
|
||||
raise unauthorized() from None
|
||||
username, _, password = decoded.partition(":")
|
||||
if not USERNAME_RE.fullmatch(username) or not valid_password(password):
|
||||
raise unauthorized()
|
||||
return username, password
|
||||
|
||||
|
||||
def password_digest(password: str, salt: bytes) -> bytes:
|
||||
return hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=32768, r=8, p=3, dklen=32,
|
||||
maxmem=64 * 1024 * 1024,
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, user: dict | None) -> bool:
|
||||
# Do equivalent KDF work for unknown usernames.
|
||||
salt, expected = (user["password_salt"], user["password_hash"]) if user else (bytes(16), bytes(32))
|
||||
match = hmac.compare_digest(password_digest(password, salt), expected)
|
||||
return user is not None and match
|
||||
|
||||
|
||||
@contextmanager
|
||||
def redis_operation():
|
||||
try:
|
||||
yield
|
||||
except RedisError:
|
||||
raise unavailable("Redis unavailable") from None
|
||||
|
||||
|
||||
class Sessions:
|
||||
def __init__(self, ttl: int):
|
||||
self.ttl = ttl
|
||||
self.redis = Redis.from_url(
|
||||
os.environ.get("REDIS_URL", "redis://redis:6379/0"),
|
||||
socket_connect_timeout=1, socket_timeout=1, max_connections=16,
|
||||
retry=Retry(NoBackoff(), 0), decode_responses=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def key(token: str) -> str:
|
||||
if not TOKEN_RE.fullmatch(token):
|
||||
raise unauthorized()
|
||||
return "lab2:session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
|
||||
def ready(self) -> None:
|
||||
with redis_operation():
|
||||
self.redis.ping()
|
||||
|
||||
def create(self, user_id: UUID) -> tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_ms = time.time_ns() // 1_000_000 + self.ttl * 1000
|
||||
with redis_operation():
|
||||
if not self.redis.set(self.key(token), str(user_id), pxat=expires_ms, nx=True):
|
||||
raise unavailable("could not create session")
|
||||
return token, datetime.fromtimestamp(expires_ms / 1000, tz=timezone.utc)
|
||||
|
||||
def identify(self, token: str) -> UUID:
|
||||
with redis_operation():
|
||||
user_id = self.redis.get(self.key(token))
|
||||
if user_id is None:
|
||||
raise unauthorized()
|
||||
return UUID(user_id)
|
||||
|
||||
def revoke(self, token: str) -> None:
|
||||
with redis_operation():
|
||||
if self.redis.delete(self.key(token)) == 0:
|
||||
raise unauthorized()
|
||||
@@ -0,0 +1,140 @@
|
||||
"""PostgreSQL storage for users, chats, membership and message history."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg import OperationalError
|
||||
from psycopg.errors import ForeignKeyViolation, LockNotAvailable, QueryCanceled
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, TooManyRequests
|
||||
|
||||
from .config import postgres_options
|
||||
from .errors import bad_request, conflict, not_found, unavailable
|
||||
from .pagination import Cursors
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, cursor_secret: bytes):
|
||||
self.cursors = Cursors(cursor_secret)
|
||||
self.pool = ConnectionPool(
|
||||
kwargs={**postgres_options(), "row_factory": dict_row},
|
||||
min_size=1, max_size=8, max_waiting=16, timeout=2, open=False,
|
||||
)
|
||||
self.pool.open()
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
# Commits on success and rolls back on any exception before reuse.
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except (OperationalError, TooManyRequests, LockNotAvailable, QueryCanceled):
|
||||
raise unavailable("PostgreSQL unavailable") from None
|
||||
|
||||
def ready(self) -> None:
|
||||
with self.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
def create_user(self, username: str, display_name: str, salt: bytes, password_hash: bytes) -> dict:
|
||||
with self.connection() as conn:
|
||||
user = conn.execute("""
|
||||
INSERT INTO users (username, display_name, password_salt, password_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (username) DO NOTHING
|
||||
RETURNING id, username, display_name, created_at
|
||||
""", (username, display_name, salt, password_hash)).fetchone()
|
||||
if user is None:
|
||||
raise conflict("username is already taken")
|
||||
return user
|
||||
|
||||
def get_user(self, user_id: UUID) -> dict | None:
|
||||
with self.connection() as conn:
|
||||
return conn.execute("""
|
||||
SELECT id, username, display_name, created_at FROM users WHERE id = %s
|
||||
""", (user_id,)).fetchone()
|
||||
|
||||
def credentials(self, username: str) -> dict | None:
|
||||
with self.connection() as conn:
|
||||
return conn.execute("""
|
||||
SELECT id, username, display_name, created_at, password_salt, password_hash
|
||||
FROM users WHERE username = %s
|
||||
""", (username,)).fetchone()
|
||||
|
||||
def create_chat(self, user_id: UUID, title: str, member_ids: list[UUID]) -> dict:
|
||||
if user_id not in member_ids:
|
||||
raise bad_request("member_ids must include the current user")
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
chat = conn.execute("""
|
||||
INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at
|
||||
""", (title,)).fetchone()
|
||||
conn.execute("""
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT %s, user_id, position
|
||||
FROM unnest(%s::uuid[]) WITH ORDINALITY AS members(user_id, position)
|
||||
""", (chat["id"], member_ids))
|
||||
except ForeignKeyViolation:
|
||||
raise bad_request("member_ids contains an unknown user") from None
|
||||
return {**chat, "member_ids": member_ids}
|
||||
|
||||
def get_chat(self, user_id: UUID, chat_id: UUID) -> dict:
|
||||
with self.connection() as conn:
|
||||
chat = conn.execute("""
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m
|
||||
WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids
|
||||
FROM chat_members member JOIN chats c ON c.id = member.chat_id
|
||||
WHERE member.user_id = %s AND c.id = %s
|
||||
""", (user_id, chat_id)).fetchone()
|
||||
if chat is None:
|
||||
raise not_found("chat not found")
|
||||
return chat
|
||||
|
||||
def list_chats(self, user_id: UUID, limit: int, cursor: str | None) -> dict:
|
||||
scope = f"chats:{user_id}"
|
||||
query = """
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m
|
||||
WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids
|
||||
FROM chat_members member JOIN chats c ON c.id = member.chat_id
|
||||
WHERE member.user_id = %s
|
||||
"""
|
||||
params = (user_id,)
|
||||
if cursor is not None:
|
||||
query += " AND (c.created_at, c.id) > (%s, %s)"
|
||||
params += self.cursors.decode(scope, cursor)
|
||||
with self.connection() as conn:
|
||||
items = conn.execute(
|
||||
query + " ORDER BY c.created_at, c.id LIMIT %s", (*params, limit + 1)
|
||||
).fetchall()
|
||||
return self.cursors.page(scope, items, limit)
|
||||
|
||||
def add_message(self, user_id: UUID, chat_id: UUID, text: str) -> dict:
|
||||
with self.connection() as conn:
|
||||
message = conn.execute("""
|
||||
INSERT INTO messages (chat_id, sender_id, text)
|
||||
SELECT chat_id, user_id, %s FROM chat_members
|
||||
WHERE chat_id = %s AND user_id = %s
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
""", (text, chat_id, user_id)).fetchone()
|
||||
if message is None:
|
||||
raise not_found("chat not found")
|
||||
return message
|
||||
|
||||
def list_messages(self, user_id: UUID, chat_id: UUID, limit: int, cursor: str | None) -> dict:
|
||||
scope = f"messages:{chat_id}"
|
||||
query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s"
|
||||
params = (chat_id,)
|
||||
with self.connection() as conn:
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||
).fetchone()
|
||||
if member is None:
|
||||
raise not_found("chat not found")
|
||||
if cursor is not None:
|
||||
query += " AND (created_at, id) > (%s, %s)"
|
||||
params += self.cursors.decode(scope, cursor)
|
||||
items = conn.execute(
|
||||
query + " ORDER BY created_at, id LIMIT %s", (*params, limit + 1)
|
||||
).fetchall()
|
||||
return self.cursors.page(scope, items, limit)
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Strict request validation: bodies, fields, path and query parameters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from .errors import bad_request
|
||||
|
||||
UUID_RE = re.compile(
|
||||
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
)
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
DIGITS_RE = re.compile(r"[0-9]{1,3}")
|
||||
|
||||
DEFAULT_LIMIT = 50
|
||||
MAX_LIMIT = 100
|
||||
MAX_CURSOR_LENGTH = 512
|
||||
|
||||
|
||||
def is_uuid(value: Any) -> bool:
|
||||
return isinstance(value, str) and UUID_RE.fullmatch(value) is not None
|
||||
|
||||
|
||||
def parse_uuid(value: Any, what: str) -> UUID:
|
||||
if not is_uuid(value):
|
||||
raise bad_request(f"{what} must be a UUID")
|
||||
return UUID(value)
|
||||
|
||||
|
||||
def _reject_constant(_name: str) -> Any:
|
||||
raise ValueError("NaN/Infinity are not valid JSON")
|
||||
|
||||
|
||||
def _object_without_duplicates(pairs: List[tuple]) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("duplicate key in JSON object")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def parse_json_object(raw: bytes, allowed: Iterable[str], required: Iterable[str]) -> Dict[str, Any]:
|
||||
"""Parse a JSON object body. Malformed JSON, extra or missing fields -> 400."""
|
||||
try:
|
||||
data = json.loads(
|
||||
raw.decode("utf-8"),
|
||||
object_pairs_hook=_object_without_duplicates,
|
||||
parse_constant=_reject_constant,
|
||||
)
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad_request("request body must be valid UTF-8 JSON") from None
|
||||
if not isinstance(data, dict):
|
||||
raise bad_request("request body must be a JSON object")
|
||||
allowed_set = set(allowed)
|
||||
if set(data) - allowed_set:
|
||||
raise bad_request("request body contains unexpected fields")
|
||||
missing = [name for name in required if name not in data]
|
||||
if missing:
|
||||
raise bad_request("missing required fields: " + ", ".join(missing))
|
||||
return data
|
||||
|
||||
|
||||
def valid_text(value: Any, minimum: int, maximum: int) -> bool:
|
||||
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
|
||||
return False
|
||||
try:
|
||||
value.encode("utf-8") # rejects lone surrogates such as "\\ud800"
|
||||
except UnicodeEncodeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def valid_password(value: Any) -> bool:
|
||||
return valid_text(value, 12, 128)
|
||||
|
||||
|
||||
def text_field(data: Dict[str, Any], name: str, max_length: int) -> str:
|
||||
value = data[name]
|
||||
if not valid_text(value, 1, max_length) or "\x00" in value:
|
||||
raise bad_request(f"{name} must contain 1..{max_length} valid nonblank characters")
|
||||
return value
|
||||
|
||||
|
||||
def username_field(data: Dict[str, Any]) -> str:
|
||||
value = data["username"]
|
||||
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
|
||||
raise bad_request("username must match [a-z0-9_]{3,32}")
|
||||
return value
|
||||
|
||||
|
||||
def member_ids_field(data: Dict[str, Any]) -> List[UUID]:
|
||||
value = data["member_ids"]
|
||||
if not isinstance(value, list) or not 1 <= len(value) <= 100:
|
||||
raise bad_request("member_ids must be an array of 1..100 user ids")
|
||||
members = [parse_uuid(item, "member_ids item") for item in value]
|
||||
if len(set(members)) != len(members):
|
||||
raise bad_request("member_ids must not contain duplicates")
|
||||
return members
|
||||
|
||||
|
||||
def _single(params: Dict[str, List[str]], name: str) -> Optional[str]:
|
||||
values = params.get(name)
|
||||
if values is None:
|
||||
return None
|
||||
if len(values) != 1:
|
||||
raise bad_request(f"{name} must be given at most once")
|
||||
return values[0]
|
||||
|
||||
|
||||
def limit_param(params: Dict[str, List[str]]) -> int:
|
||||
raw = _single(params, "limit")
|
||||
if raw is None:
|
||||
return DEFAULT_LIMIT
|
||||
if DIGITS_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
|
||||
raise bad_request(f"limit must be an integer 1..{MAX_LIMIT}")
|
||||
return int(raw)
|
||||
|
||||
|
||||
def cursor_param(params: Dict[str, List[str]]) -> Optional[str]:
|
||||
raw = _single(params, "cursor")
|
||||
if raw is None:
|
||||
return None
|
||||
if not raw or len(raw) > MAX_CURSOR_LENGTH:
|
||||
raise bad_request("invalid cursor")
|
||||
return raw
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Check grade 4 against the local Compose stand using synthetic data."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from threading import Barrier
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, arguments, check, run, timestamp
|
||||
|
||||
LOGIN = "/api/v1/auth/sessions"
|
||||
ME = "/api/v1/users/me"
|
||||
|
||||
|
||||
def compose(*args):
|
||||
result = subprocess.run(
|
||||
["docker", "compose", *args], cwd=ROOT, check=True, timeout=60,
|
||||
stdout=subprocess.PIPE, text=True, encoding="utf-8",
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def basic(registration, password):
|
||||
pair = f"{registration['username']}:{password}".encode()
|
||||
return {"Authorization": "Basic " + base64.b64encode(pair).decode()}
|
||||
|
||||
|
||||
def race(client):
|
||||
count = 6
|
||||
body = {
|
||||
"username": "race_" + uuid.uuid4().hex[:20],
|
||||
"display_name": "Гонка регистрации",
|
||||
"password": "Синтетический:" + uuid.uuid4().hex,
|
||||
}
|
||||
barrier = Barrier(count, timeout=10)
|
||||
|
||||
def attempt(_):
|
||||
worker = Client(client.base)
|
||||
barrier.wait()
|
||||
return worker.call("POST", "/api/v1/users", (201, 409), body=body)["status"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as workers:
|
||||
statuses = sorted(workers.map(attempt, range(count)))
|
||||
check(statuses == [201] + [409] * (count - 1), "duplicate registration race failed")
|
||||
print(f"PASS registration race ({count} concurrent attempts): {statuses}")
|
||||
client.call("POST", LOGIN, 201, schema="Session", headers=basic(body, body["password"]))
|
||||
print("PASS Basic login with a Unicode password containing a colon")
|
||||
|
||||
|
||||
def rollback(client, actor):
|
||||
# The title is generated here from fixed ASCII and a UUID, with no external input.
|
||||
title = "rollback_" + uuid.uuid4().hex
|
||||
client.call("POST", "/api/v1/chats", 400, actor=actor, body={
|
||||
"title": title, "member_ids": [actor["id"], str(uuid.uuid4())],
|
||||
})
|
||||
remaining = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-XAt", "-v", "ON_ERROR_STOP=1", "-c",
|
||||
f"SELECT count(*) FROM chats WHERE title = '{title}'")
|
||||
check(remaining == "0", "a rejected chat left an orphan row")
|
||||
print("PASS rollback: HTTP 400 and zero rows for rejected chat in PostgreSQL")
|
||||
|
||||
|
||||
def wait_ready(client):
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if client.call("GET", "/health/ready", (200, 503))["status"] == 200:
|
||||
return
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
raise Failure("API did not become ready")
|
||||
|
||||
|
||||
def outage(client, service, actor):
|
||||
compose("stop", service)
|
||||
try:
|
||||
client.call("GET", "/health/live", schema="Health")
|
||||
client.call("GET", "/health/ready", 503)
|
||||
failure = client.call("GET", ME, 503, actor=actor)
|
||||
check(failure["elapsed"] < 5, "dependency failure exceeded HTTP time budget")
|
||||
client.call("GET", ME, 401, headers={"X-User-Id": actor["id"]})
|
||||
finally:
|
||||
compose("start", service)
|
||||
started = time.monotonic()
|
||||
wait_ready(client)
|
||||
same = client.call("GET", ME, actor=actor, schema="User")["json"]
|
||||
check(same == actor["user"], "previous session did not recover")
|
||||
print(f"PASS {service} outage: live=200 ready=503 Bearer=503 "
|
||||
f"({failure['elapsed']:.3f}s) anonymous=401; same token after start "
|
||||
f"({time.monotonic() - started:.3f}s from docker compose start return)")
|
||||
|
||||
|
||||
def expiry(client):
|
||||
actor = client.user("expiry")
|
||||
session = client.call("POST", LOGIN, 201, schema="Session",
|
||||
headers=basic(actor["registration"], actor["registration"]["password"]))["json"]
|
||||
actor["token"] = session["token"]
|
||||
remaining = (timestamp(session["expires_at"]) - datetime.now(timezone.utc)).total_seconds()
|
||||
check(0 < remaining <= 2.1, "start API with SESSION_TTL_SECONDS=2 for this test")
|
||||
client.call("GET", ME, actor=actor, schema="User")
|
||||
time.sleep(remaining + 0.15)
|
||||
client.call("GET", ME, 401, actor=actor)
|
||||
print("PASS 2s expiry: same token 200 before deadline and 401 after deadline")
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab 2 grade 4 checks (stop and restore dependencies)",
|
||||
lambda p: p.add_argument("--expiry-only", action="store_true"))
|
||||
client = Client(args.base_url, args.ca_file, timeout=6)
|
||||
wait_ready(client)
|
||||
if args.expiry_only:
|
||||
expiry(client)
|
||||
return
|
||||
race(client)
|
||||
actor = client.user("outage")
|
||||
rollback(client, actor)
|
||||
client.call("POST", LOGIN, 401, headers=basic(actor["registration"], "Wrong:password-123"))
|
||||
revoked = client.user("logout")
|
||||
client.call("DELETE", LOGIN + "/current", 204, actor=revoked)
|
||||
client.call("GET", ME, 401, actor=revoked)
|
||||
print("PASS wrong password=401, logout=204, revoked token=401")
|
||||
key = "lab2:session:" + hashlib.sha256(actor["token"].encode()).hexdigest()
|
||||
ttl = int(compose("exec", "-T", "redis", "redis-cli", "TTL", key))
|
||||
check(ttl >= 300, "start the default-TTL stand for the dependency tests")
|
||||
print(f"PASS Redis session key has TTL ({ttl}s left)")
|
||||
outage(client, "redis", actor)
|
||||
client.call("GET", ME, 401, actor=revoked)
|
||||
print("PASS Redis restart did not revive revoked session")
|
||||
outage(client, "postgres", actor)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user