forked from mtusi2026/mtusi2026lab2
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
388f31c5f4 |
@@ -18,3 +18,4 @@ __pycache__
|
|||||||
node_modules
|
node_modules
|
||||||
evidence
|
evidence
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
**/__pycache__
|
||||||
|
|||||||
@@ -1,18 +1,13 @@
|
|||||||
## Лабораторная 2
|
## Лабораторная 2. PostgreSQL и сессии
|
||||||
|
|
||||||
Автор / группа: TODO
|
Автор и группа — Samodurov Nikita, bpi2303
|
||||||
Целевая оценка (3/4/5): TODO
|
|
||||||
Трек 5, если заявлен: TODO
|
|
||||||
|
|
||||||
Что реализовано и что перенесено: TODO
|
Целевая оценка — 4
|
||||||
Команды запуска из чистого clone: TODO
|
|
||||||
Результаты проверок и ссылка на REPORT.md: TODO
|
|
||||||
Ограничения / известные ошибки: TODO
|
|
||||||
Источники и использование ИИ: TODO
|
|
||||||
|
|
||||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
Пользователи, чаты, участники, сообщения и сессии хранятся в PostgreSQL. Вход создаёт Bearer-сессию с TTL, logout отзывает её. Redis кэширует неизменяемый профиль.
|
||||||
- [ ] Контракт и публичные тесты не ослаблены.
|
|
||||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
Команды запуска и ограничения описаны в [REPORT.md](REPORT.md). Результаты smoke, перезапуска и дополнительных сценариев сохранены в [evidence/local-checks.txt](evidence/local-checks.txt).
|
||||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
|
||||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
Источники — материалы и контракт курса.
|
||||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
|
||||||
|
Использование ИИ — консультации и проверки ИИ.
|
||||||
|
|||||||
@@ -12,5 +12,29 @@ jobs:
|
|||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
- run: python -m pip install -r requirements-dev.txt
|
- run: python -m pip install -r requirements-dev.txt
|
||||||
- run: make check validate PYTHON=python
|
- run: make check validate PYTHON=python
|
||||||
# This job validates the skeleton, not a student solution.
|
application:
|
||||||
# Add build, readiness and make test in a separate application job.
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.13'
|
||||||
|
- name: Build and start stack
|
||||||
|
run: |
|
||||||
|
python scripts/setup.py
|
||||||
|
chmod +x scripts/restart.sh
|
||||||
|
docker compose build migrate
|
||||||
|
docker compose up -d --wait
|
||||||
|
- name: Run HTTP and resilience checks
|
||||||
|
run: |
|
||||||
|
make test
|
||||||
|
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
python student-tests/check_dependencies.py
|
||||||
|
docker compose exec -T db psql -U messenger -d messenger < student-tests/explain.sql
|
||||||
|
- name: Show service status
|
||||||
|
if: failure()
|
||||||
|
run: docker compose ps -a
|
||||||
|
- name: Stop stack
|
||||||
|
if: always()
|
||||||
|
run: docker compose down
|
||||||
|
|||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
FROM python:3.13.5-slim-bookworm@sha256:4c2cf9917bd1cbacc5e9b07320025bdb7cdf2df7b0ceaccb55e9dd7e30987419
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||||
|
WORKDIR /app
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
COPY migrations/ ./migrations/
|
||||||
|
COPY src/ ./src/
|
||||||
|
USER 10001:10001
|
||||||
|
EXPOSE 8080
|
||||||
|
HEALTHCHECK --interval=3s --timeout=3s --start-period=5s --retries=4 \
|
||||||
|
CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=2).close()"]
|
||||||
|
CMD ["python", "src/server.py"]
|
||||||
@@ -1,37 +1,53 @@
|
|||||||
# Отчёт — лабораторная 2
|
# Отчёт — лабораторная 2
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор и группа — Samodurov Nikita, bpi2303
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка — 4
|
||||||
- Commit/PR: TODO после публикации
|
- Источники — материалы и контракт курса.
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- Использование ИИ — консультации и проверки ИИ.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого клона
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
Нужны Docker Engine с Compose v2 и Python 3.9+ на хосте. Контейнер использует Python 3.13.5. По умолчанию API доступен на http://127.0.0.1:8080. При занятом порте задайте `PUBLIC_PORT` и соответствующий `BASE_URL`. Команда setup создаёт локальные `.env` и `secrets/`.
|
||||||
|
|
||||||
## Архитектура и решения
|
```bash
|
||||||
|
python scripts/setup.py
|
||||||
|
make check
|
||||||
|
docker compose config --quiet
|
||||||
|
docker compose build migrate
|
||||||
|
docker compose up -d --wait
|
||||||
|
make test
|
||||||
|
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
python student-tests/check_dependencies.py
|
||||||
|
docker compose exec -T db psql -U messenger -d messenger < student-tests/explain.sql
|
||||||
|
docker compose down
|
||||||
|
```
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
В Windows PowerShell используйте следующие проверки после запуска стенда. Перед проверками установите `$env:PYTHONUTF8 = '1'`.
|
||||||
|
|
||||||
## Доказательства критериев
|
```powershell
|
||||||
|
python scripts/check_contract.py
|
||||||
|
python tests/smoke.py
|
||||||
|
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||||
|
python student-tests/check_dependencies.py
|
||||||
|
Get-Content student-tests/explain.sql -Raw | docker compose exec -T db psql -U messenger -d messenger
|
||||||
|
```
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
Для полной валидации контракта установите `requirements-dev.txt` в отдельное окружение и выполните `python scripts/validate_openapi.py`. Команда `docker compose down` оставляет именованные тома. Для намеренного сброса учебных данных выполните `docker compose down -v`.
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
## Архитектура
|
||||||
|
|
||||||
## Эксперименты и отказы
|
Из ЛР 1 перенесены HTTP-обработчик, проверки полей, логика доступа и курсорная пагинация. Стенд состоит из API, PostgreSQL и Redis. Пароли хешируются scrypt с индивидуальной солью. В PostgreSQL хранится SHA-256 от случайного 256-битного токена. Срок сессии проверяется по часам PostgreSQL. Заголовок X-User-Id не участвует в аутентификации.
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
Уникальность username обеспечивает индекс PostgreSQL. Чат и его участники записываются в одной транзакции. Внешние ключи не допускают участников вне списка пользователей и сообщения от пользователя вне чата. SQL использует параметры. Пул ограничен восемью соединениями на процесс, ожидание свободного соединения ограничено 1,5 с, выполнение SQL — 3 с.
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
PostgreSQL остаётся источником истины. После проверки сессии запрос к /users/me может взять неизменяемый профиль из Redis с TTL 300 с. При задержке свыше 350 мс API читает профиль из PostgreSQL. Очередь кэша ограничена четырьмя операциями. Изменение профиля не входит в API этой работы.
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| TODO | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
История использует составной индекс (chat_id, created_at, id) и курсор по паре (created_at, id). Подпись курсора проверяется общим ключом из secrets. Между страницами возможны конкурентные вставки, поэтому обход не фиксирует снимок истории. Поздно закоммиченная запись с ключом до курсора видна при новом обходе.
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
## Проверки
|
||||||
|
|
||||||
|
Результаты локального Docker-стенда и команды приведены в [evidence/local-checks.txt](evidence/local-checks.txt).
|
||||||
|
|
||||||
|
## Ограничения
|
||||||
|
|
||||||
|
При остановке PostgreSQL защищённый API недоступен. При отключении Redis профили читаются из PostgreSQL. Рестарт сохраняет данные и активные сессии при сохранённом томе pgdata.
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
x-app: &app
|
||||||
|
image: mtusi-messenger:lab2
|
||||||
|
env_file: .env
|
||||||
|
environment:
|
||||||
|
CURSOR_KEY_FILE: /run/secrets/cursor_key
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
secrets: [database_url, cache_url, cursor_key]
|
||||||
|
|
||||||
|
services:
|
||||||
|
db:
|
||||||
|
image: postgres:17.5-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: messenger
|
||||||
|
POSTGRES_DB: messenger
|
||||||
|
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||||
|
secrets: [db_password]
|
||||||
|
volumes: [pgdata:/var/lib/postgresql/data]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 20
|
||||||
|
|
||||||
|
cache:
|
||||||
|
image: redis:7.4.3-alpine
|
||||||
|
command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "32mb", "--maxmemory-policy", "allkeys-lru"]
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
<<: *app
|
||||||
|
build: .
|
||||||
|
command: ["python", "src/migrate.py"]
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
depends_on:
|
||||||
|
db:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
|
api:
|
||||||
|
<<: *app
|
||||||
|
ports: ["127.0.0.1:${PUBLIC_PORT:-8080}:${APP_PORT:-8080}"]
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
cache:
|
||||||
|
condition: service_started
|
||||||
|
stop_grace_period: 8s
|
||||||
|
|
||||||
|
secrets:
|
||||||
|
db_password:
|
||||||
|
file: ./secrets/db_password
|
||||||
|
database_url:
|
||||||
|
file: ./secrets/database_url
|
||||||
|
cache_url:
|
||||||
|
file: ./secrets/cache_url
|
||||||
|
cursor_key:
|
||||||
|
file: ./secrets/cursor_key
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
pgdata:
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
2026-11-12. Windows, Python 3.13.5, Docker Desktop с Linux-контейнерами.
|
||||||
|
PUBLIC_PORT=18082, BASE_URL=http://127.0.0.1:18082
|
||||||
|
|
||||||
|
> python scripts/check_contract.py
|
||||||
|
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||||
|
|
||||||
|
> python scripts/validate_openapi.py
|
||||||
|
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||||
|
|
||||||
|
> python tests/smoke.py --base-url http://127.0.0.1:18082
|
||||||
|
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
|
|
||||||
|
> python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||||
|
Hook: docker compose restart db cache api.
|
||||||
|
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
|
||||||
|
> python student-tests/check_dependencies.py
|
||||||
|
PASS: Redis contains the profile entry
|
||||||
|
PASS: wrong password, X-User-Id rejected, concurrent registration, chat rollback
|
||||||
|
PASS: cache stopped, authenticated GET returned 200
|
||||||
|
PASS: db stopped, authenticated GET returned 503
|
||||||
|
PASS: logout and 2-second session expiry
|
||||||
|
|
||||||
|
> Get-Content student-tests/explain.sql -Raw | docker compose exec -T db psql -U messenger -d messenger
|
||||||
|
INSERT 0 12000
|
||||||
|
ANALYZE
|
||||||
|
Limit (actual time=0.024..0.033 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=3
|
||||||
|
-> Index Scan using messages_history_idx on messages (actual time=0.024..0.030 rows=51 loops=1)
|
||||||
|
Index Cond: (chat_id = … AND ROW(created_at, id) > ROW(…, …))
|
||||||
|
Buffers: shared hit=3
|
||||||
|
Planning Time: 0.147 ms
|
||||||
|
Execution Time: 0.126 ms
|
||||||
|
ROLLBACK
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
CREATE TABLE users (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||||
|
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||||
|
password_hash bytea NOT NULL,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE members (
|
||||||
|
chat_id uuid REFERENCES chats(id),
|
||||||
|
user_id uuid REFERENCES users(id),
|
||||||
|
position integer NOT NULL,
|
||||||
|
PRIMARY KEY (chat_id, user_id),
|
||||||
|
UNIQUE (chat_id, position)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX members_user_idx ON members (user_id, chat_id);
|
||||||
|
CREATE INDEX chats_order_idx ON chats (created_at, id);
|
||||||
|
|
||||||
|
CREATE TABLE messages (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
chat_id uuid NOT NULL,
|
||||||
|
sender_id uuid NOT NULL,
|
||||||
|
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
FOREIGN KEY (chat_id, sender_id) REFERENCES members (chat_id, user_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||||
|
|
||||||
|
CREATE TABLE sessions (
|
||||||
|
token_hash bytea PRIMARY KEY,
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
expires_at timestamptz NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
psycopg==3.2.9
|
||||||
|
psycopg-binary==3.2.9
|
||||||
|
psycopg-pool==3.2.6
|
||||||
|
redis==6.2.0
|
||||||
|
typing-extensions==4.14.0
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
@echo off
|
||||||
|
cd /d "%~dp0.."
|
||||||
|
docker compose restart db cache api
|
||||||
Executable
+4
@@ -0,0 +1,4 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
docker compose restart db cache api
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
root = Path(__file__).resolve().parents[1]
|
||||||
|
directory = root / "secrets"
|
||||||
|
directory.mkdir(mode=0o700, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure(name, value):
|
||||||
|
path = directory / name
|
||||||
|
try:
|
||||||
|
with path.open("x", encoding="utf-8") as file:
|
||||||
|
file.write(value + "\n")
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
return path.read_text(encoding="utf-8").strip()
|
||||||
|
|
||||||
|
|
||||||
|
password = ensure("db_password", secrets.token_hex(24))
|
||||||
|
ensure("database_url", f"postgresql://messenger:{password}@db:5432/messenger")
|
||||||
|
ensure("cache_url", "redis://cache:6379/0")
|
||||||
|
ensure("cursor_key", secrets.token_hex(32))
|
||||||
|
|
||||||
|
try:
|
||||||
|
with (root / ".env").open("xb") as file:
|
||||||
|
file.write((root / ".env.example").read_bytes())
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def secret(name):
|
||||||
|
return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def positive_int(name, default):
|
||||||
|
value = int(os.getenv(name, default))
|
||||||
|
if value < 1:
|
||||||
|
raise ValueError(f"{name} must be positive")
|
||||||
|
return value
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import psycopg
|
||||||
|
from config import secret
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=3) as connection:
|
||||||
|
connection.execute("SELECT pg_advisory_xact_lock(230301)")
|
||||||
|
connection.execute("CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY)")
|
||||||
|
for path in sorted(Path("migrations").glob("*.sql")):
|
||||||
|
if connection.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone():
|
||||||
|
continue
|
||||||
|
connection.execute(path.read_text(encoding="utf-8"))
|
||||||
|
connection.execute("INSERT INTO schema_migrations (name) VALUES (%s)", (path.name,))
|
||||||
|
print(f"Applied {path.name}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
from concurrent.futures import Future, TimeoutError
|
||||||
|
from queue import Empty, Full, Queue
|
||||||
|
from threading import Event, Thread
|
||||||
|
|
||||||
|
import redis
|
||||||
|
from redis.backoff import NoBackoff
|
||||||
|
from redis.retry import Retry
|
||||||
|
|
||||||
|
|
||||||
|
class ProfileCache:
|
||||||
|
def __init__(self, url):
|
||||||
|
self.client = redis.Redis.from_url(
|
||||||
|
url, socket_timeout=0.25, socket_connect_timeout=0.25,
|
||||||
|
max_connections=1, retry=Retry(NoBackoff(), 0),
|
||||||
|
)
|
||||||
|
self.work = Queue(maxsize=4)
|
||||||
|
self.stopping = Event()
|
||||||
|
Thread(target=self.run, daemon=True, name="profile-cache").start()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.stopping.set()
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
try:
|
||||||
|
while not self.stopping.is_set():
|
||||||
|
try:
|
||||||
|
result, operation, args = self.work.get(timeout=0.25)
|
||||||
|
except Empty:
|
||||||
|
continue
|
||||||
|
if not result.set_running_or_notify_cancel():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
result.set_result(operation(*args))
|
||||||
|
except redis.RedisError:
|
||||||
|
result.set_result(None)
|
||||||
|
finally:
|
||||||
|
self.client.close()
|
||||||
|
|
||||||
|
def submit(self, operation, *args):
|
||||||
|
result = Future()
|
||||||
|
try:
|
||||||
|
self.work.put_nowait((result, operation, args))
|
||||||
|
except Full:
|
||||||
|
result.set_result(None)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def get(self, key):
|
||||||
|
result = self.submit(self.client.get, key)
|
||||||
|
try:
|
||||||
|
return result.result(timeout=0.35)
|
||||||
|
except TimeoutError:
|
||||||
|
result.cancel()
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set(self, key, value):
|
||||||
|
self.submit(self.client.setex, key, 300, value)
|
||||||
+288
@@ -0,0 +1,288 @@
|
|||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import signal
|
||||||
|
import threading
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from urllib.parse import parse_qs, unquote, urlsplit
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from storage import DB_UNAVAILABLE, Store, json_value, unauthorized, unavailable
|
||||||
|
from validation import ApiError, canonical_id, invalid, not_found, require_fields, require_text
|
||||||
|
|
||||||
|
CHAT_PATH = re.compile(r"/api/v1/chats/([^/]+)(/messages)?")
|
||||||
|
MAX_JSON_BYTES = 64 * 1024
|
||||||
|
LOG = logging.getLogger("messenger")
|
||||||
|
|
||||||
|
|
||||||
|
def unique_object(pairs):
|
||||||
|
result = dict(pairs)
|
||||||
|
if len(result) != len(pairs):
|
||||||
|
raise ValueError
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def page_query(query):
|
||||||
|
try:
|
||||||
|
params = parse_qs(query, keep_blank_values=True, max_num_fields=16)
|
||||||
|
except ValueError:
|
||||||
|
raise invalid("Invalid query") from None
|
||||||
|
limits = params.get("limit", ["50"])
|
||||||
|
cursors = params.get("cursor", [None])
|
||||||
|
if len(limits) != 1 or not re.fullmatch(r"[0-9]{1,3}", limits[0]):
|
||||||
|
raise invalid("Invalid limit")
|
||||||
|
limit = int(limits[0])
|
||||||
|
if not 1 <= limit <= 100:
|
||||||
|
raise invalid("Invalid limit")
|
||||||
|
if len(cursors) != 1:
|
||||||
|
raise invalid("Invalid cursor")
|
||||||
|
return limit, cursors[0]
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
self.handle_api()
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
self.handle_api()
|
||||||
|
|
||||||
|
def do_DELETE(self):
|
||||||
|
self.handle_api()
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def log_request(self, code="-", size="-"):
|
||||||
|
LOG.info("request_id=%s status=%s", self.request_id, code)
|
||||||
|
|
||||||
|
def send_error(self, code, message=None, explain=None):
|
||||||
|
self.request_id = uuid4().hex
|
||||||
|
self.response_headers = {}
|
||||||
|
self.send_json(code, {"error": {
|
||||||
|
"code": "invalid_request",
|
||||||
|
"message": "Invalid HTTP request",
|
||||||
|
"request_id": self.request_id,
|
||||||
|
}})
|
||||||
|
|
||||||
|
def handle_api(self):
|
||||||
|
self.request_id = uuid4().hex
|
||||||
|
self.response_headers = {}
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
status, payload = self.route()
|
||||||
|
except ApiError as error:
|
||||||
|
status, payload = self.error_payload(error)
|
||||||
|
except DB_UNAVAILABLE:
|
||||||
|
status, payload = self.error_payload(unavailable())
|
||||||
|
except (TimeoutError, ConnectionError):
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
LOG.error("request_id=%s unexpected error", self.request_id)
|
||||||
|
status, payload = self.error_payload(
|
||||||
|
ApiError(500, "internal_error", "Internal server error")
|
||||||
|
)
|
||||||
|
self.send_json(status, payload)
|
||||||
|
except (TimeoutError, ConnectionError):
|
||||||
|
return
|
||||||
|
|
||||||
|
def error_payload(self, error):
|
||||||
|
self.response_headers = error.headers
|
||||||
|
return error.status, {"error": {
|
||||||
|
"code": error.code,
|
||||||
|
"message": str(error),
|
||||||
|
"request_id": self.request_id,
|
||||||
|
}}
|
||||||
|
|
||||||
|
def send_json(self, status, payload):
|
||||||
|
data = b"" if status == 204 else json.dumps(payload, ensure_ascii=False, default=json_value).encode("utf-8")
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("X-Request-Id", self.request_id)
|
||||||
|
if status != 204:
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
for name, value in self.response_headers.items():
|
||||||
|
self.send_header(name, value)
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
if self.command != "HEAD":
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
def read_json(self):
|
||||||
|
lengths = self.headers.get_all("Content-Length", [])
|
||||||
|
if (
|
||||||
|
self.headers.get("Transfer-Encoding")
|
||||||
|
or len(lengths) != 1
|
||||||
|
or not re.fullmatch(r"[0-9]{1,10}", lengths[0])
|
||||||
|
or self.headers.get_content_type() != "application/json"
|
||||||
|
):
|
||||||
|
raise invalid("Expected a JSON body")
|
||||||
|
length = int(lengths[0])
|
||||||
|
if length > MAX_JSON_BYTES:
|
||||||
|
raise invalid("JSON body is too large")
|
||||||
|
try:
|
||||||
|
data = self.rfile.read(length)
|
||||||
|
if len(data) != length:
|
||||||
|
raise ValueError
|
||||||
|
return json.loads(data.decode("utf-8"), object_pairs_hook=unique_object)
|
||||||
|
except (ValueError, RecursionError):
|
||||||
|
raise invalid("Invalid JSON") from None
|
||||||
|
|
||||||
|
def route(self):
|
||||||
|
try:
|
||||||
|
target = urlsplit(self.path)
|
||||||
|
except ValueError:
|
||||||
|
raise invalid("Invalid request path") from None
|
||||||
|
path, method = target.path, self.command
|
||||||
|
app = self.server.app
|
||||||
|
|
||||||
|
if method == "GET" and path == "/health/live":
|
||||||
|
return 200, {"status": "ok"}
|
||||||
|
if method == "GET" and path == "/health/ready":
|
||||||
|
app.ready()
|
||||||
|
return 200, {"status": "ok"}
|
||||||
|
if method == "POST" and path == "/api/v1/users":
|
||||||
|
body = self.read_json()
|
||||||
|
require_fields(body, "username", "display_name", "password")
|
||||||
|
username = body["username"]
|
||||||
|
if not isinstance(username, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||||
|
raise invalid("Invalid username")
|
||||||
|
password = body["password"]
|
||||||
|
if not isinstance(password, str) or not 12 <= len(password) <= 128:
|
||||||
|
raise invalid("Invalid password")
|
||||||
|
try:
|
||||||
|
password.encode("utf-8")
|
||||||
|
except UnicodeError:
|
||||||
|
raise invalid("Invalid password") from None
|
||||||
|
return 201, app.register(username, require_text(body["display_name"], 100), password)
|
||||||
|
if method == "POST" and path == "/api/v1/auth/sessions":
|
||||||
|
return 201, app.login(*self.basic_credentials())
|
||||||
|
|
||||||
|
match = CHAT_PATH.fullmatch(path)
|
||||||
|
if path not in ("/api/v1/users/me", "/api/v1/chats", "/api/v1/auth/sessions/current") and not match:
|
||||||
|
raise not_found()
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
bearer = re.fullmatch(r"(?i:Bearer) ([A-Za-z0-9_-]{43})", header)
|
||||||
|
if not bearer:
|
||||||
|
raise unauthorized("Bearer")
|
||||||
|
digest = app.token_hash(bearer[1])
|
||||||
|
actor = app.actor(digest)
|
||||||
|
if path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
||||||
|
app.logout(digest)
|
||||||
|
return 204, None
|
||||||
|
if path == "/api/v1/users/me" and method == "GET":
|
||||||
|
return 200, app.user(actor)
|
||||||
|
if path == "/api/v1/chats":
|
||||||
|
if method == "POST":
|
||||||
|
body = self.read_json()
|
||||||
|
require_fields(body, "title", "member_ids")
|
||||||
|
title = require_text(body["title"], 100)
|
||||||
|
members = body["member_ids"]
|
||||||
|
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||||
|
raise invalid("Invalid member list")
|
||||||
|
members = [canonical_id(member) for member in members]
|
||||||
|
if actor not in members or len(set(members)) != len(members):
|
||||||
|
raise invalid("Invalid member list")
|
||||||
|
return 201, app.create_chat(title, members)
|
||||||
|
if method == "GET":
|
||||||
|
return 200, app.chat_page(actor, *page_query(target.query))
|
||||||
|
if match:
|
||||||
|
chat_id = canonical_id(unquote(match[1]))
|
||||||
|
chat = app.chat(actor, chat_id)
|
||||||
|
if match[2]:
|
||||||
|
if method == "POST":
|
||||||
|
body = self.read_json()
|
||||||
|
require_fields(body, "text")
|
||||||
|
return 201, app.send(actor, chat_id, require_text(body["text"], 4000))
|
||||||
|
if method == "GET":
|
||||||
|
return 200, app.message_page(chat_id, *page_query(target.query))
|
||||||
|
elif method == "GET":
|
||||||
|
return 200, chat
|
||||||
|
raise not_found()
|
||||||
|
|
||||||
|
def basic_credentials(self):
|
||||||
|
if self.headers.get("Content-Length", "0") != "0" or self.headers.get("Transfer-Encoding"):
|
||||||
|
raise invalid("Login does not accept a body")
|
||||||
|
match = re.fullmatch(r"(?i:Basic) ([A-Za-z0-9+/=]{1,800})",
|
||||||
|
self.headers.get("Authorization", ""))
|
||||||
|
if not match:
|
||||||
|
raise unauthorized("Basic")
|
||||||
|
try:
|
||||||
|
value = base64.b64decode(match[1], validate=True).decode("utf-8")
|
||||||
|
except ValueError:
|
||||||
|
raise unauthorized("Basic") from None
|
||||||
|
username, separator, password = value.partition(":")
|
||||||
|
if not separator or not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
|
||||||
|
raise unauthorized("Basic")
|
||||||
|
return username, password
|
||||||
|
|
||||||
|
|
||||||
|
class Server(ThreadingHTTPServer):
|
||||||
|
timeout = 0.25
|
||||||
|
request_queue_size = 64
|
||||||
|
|
||||||
|
def __init__(self, address):
|
||||||
|
self.app = Store()
|
||||||
|
self.stopping = False
|
||||||
|
self.active = 0
|
||||||
|
self.workers = threading.Condition()
|
||||||
|
super().__init__(address, Handler)
|
||||||
|
|
||||||
|
def get_request(self):
|
||||||
|
connection, address = super().get_request()
|
||||||
|
connection.settimeout(4)
|
||||||
|
return connection, address
|
||||||
|
|
||||||
|
def verify_request(self, request, client_address):
|
||||||
|
return not self.stopping
|
||||||
|
|
||||||
|
def process_request(self, request, client_address):
|
||||||
|
with self.workers:
|
||||||
|
self.active += 1
|
||||||
|
try:
|
||||||
|
super().process_request(request, client_address)
|
||||||
|
except Exception:
|
||||||
|
self.worker_done()
|
||||||
|
raise
|
||||||
|
|
||||||
|
def process_request_thread(self, request, client_address):
|
||||||
|
try:
|
||||||
|
super().process_request_thread(request, client_address)
|
||||||
|
finally:
|
||||||
|
self.worker_done()
|
||||||
|
|
||||||
|
def worker_done(self):
|
||||||
|
with self.workers:
|
||||||
|
self.active -= 1
|
||||||
|
self.workers.notify_all()
|
||||||
|
|
||||||
|
def handle_error(self, request, client_address):
|
||||||
|
LOG.error("HTTP connection handler failed")
|
||||||
|
|
||||||
|
def stop(self, signum, frame):
|
||||||
|
self.stopping = True
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
while not self.stopping:
|
||||||
|
self.handle_request()
|
||||||
|
self.server_close()
|
||||||
|
with self.workers:
|
||||||
|
self.workers.wait_for(lambda: self.active == 0, timeout=5)
|
||||||
|
self.app.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
host = os.getenv("APP_HOST", "0.0.0.0")
|
||||||
|
port = int(os.getenv("APP_PORT", "8080"))
|
||||||
|
environment = os.getenv("APP_ENV", "lab")
|
||||||
|
with Server((host, port)) as server:
|
||||||
|
signal.signal(signal.SIGTERM, server.stop)
|
||||||
|
signal.signal(signal.SIGINT, server.stop)
|
||||||
|
LOG.info("started host=%s port=%s env=%s", host, server.server_port, environment)
|
||||||
|
server.run()
|
||||||
|
LOG.info("stopped")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+211
@@ -0,0 +1,211 @@
|
|||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import secrets
|
||||||
|
from datetime import datetime
|
||||||
|
from threading import BoundedSemaphore
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||||
|
|
||||||
|
from config import positive_int, secret
|
||||||
|
from profile_cache import ProfileCache
|
||||||
|
from validation import ApiError, invalid, not_found
|
||||||
|
|
||||||
|
USER = "id, username, display_name, created_at"
|
||||||
|
MESSAGE = "id, chat_id, sender_id, text, created_at"
|
||||||
|
CHAT = """c.id, c.title, c.created_at,
|
||||||
|
ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
|
||||||
|
DB_UNAVAILABLE = (psycopg.OperationalError, PoolTimeout, psycopg.errors.QueryCanceled,
|
||||||
|
psycopg.errors.LockNotAvailable)
|
||||||
|
LOG = logging.getLogger("messenger")
|
||||||
|
|
||||||
|
|
||||||
|
def json_value(value):
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||||
|
if isinstance(value, UUID):
|
||||||
|
return str(value)
|
||||||
|
raise TypeError(type(value).__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def unavailable():
|
||||||
|
return ApiError(503, "unavailable", "Dependency temporarily unavailable")
|
||||||
|
|
||||||
|
|
||||||
|
def unauthorized(scheme):
|
||||||
|
challenge = f'{scheme} realm="messenger"'
|
||||||
|
if scheme == "Basic":
|
||||||
|
challenge += ', charset="UTF-8"'
|
||||||
|
return ApiError(401, "unauthorized", "Invalid credentials", {"WWW-Authenticate": challenge})
|
||||||
|
|
||||||
|
|
||||||
|
class Store:
|
||||||
|
def __init__(self):
|
||||||
|
self.pool = ConnectionPool(
|
||||||
|
secret("DATABASE_URL"), min_size=1, max_size=8, max_waiting=24, timeout=1.5,
|
||||||
|
kwargs={"row_factory": dict_row, "connect_timeout": 2,
|
||||||
|
"options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"},
|
||||||
|
open=True, check=ConnectionPool.check_connection,
|
||||||
|
)
|
||||||
|
self.cache = ProfileCache(secret("CACHE_URL"))
|
||||||
|
self.cursor_key = bytes.fromhex(secret("CURSOR_KEY"))
|
||||||
|
self.ttl = positive_int("SESSION_TTL_SECONDS", 3600)
|
||||||
|
self.hash_slots = BoundedSemaphore(4)
|
||||||
|
self.dummy_hash = bytes(16 + 32)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.cache.close()
|
||||||
|
self.pool.close()
|
||||||
|
|
||||||
|
def ready(self):
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
db.execute("SELECT 1 FROM users LIMIT 1")
|
||||||
|
|
||||||
|
def password_hash(self, password, salt):
|
||||||
|
if not self.hash_slots.acquire(timeout=0.5):
|
||||||
|
raise unavailable()
|
||||||
|
try:
|
||||||
|
return salt + hashlib.scrypt(
|
||||||
|
password.encode("utf-8"), salt=salt, n=2**15, r=8, p=1,
|
||||||
|
maxmem=64 * 1024**2, dklen=32,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
self.hash_slots.release()
|
||||||
|
|
||||||
|
def register(self, username, name, password):
|
||||||
|
password_hash = self.password_hash(password, secrets.token_bytes(16))
|
||||||
|
try:
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
return db.execute(f"""
|
||||||
|
INSERT INTO users (username,display_name,password_hash) VALUES (%s,%s,%s)
|
||||||
|
RETURNING {USER}
|
||||||
|
""", (username, name, password_hash)).fetchone()
|
||||||
|
except psycopg.errors.UniqueViolation:
|
||||||
|
raise ApiError(409, "conflict", "Username already exists") from None
|
||||||
|
|
||||||
|
def login(self, username, password):
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
row = db.execute(f"SELECT {USER},password_hash FROM users WHERE username=%s",
|
||||||
|
(username,)).fetchone()
|
||||||
|
stored = row.pop("password_hash") if row else self.dummy_hash
|
||||||
|
matches = hmac.compare_digest(self.password_hash(password, stored[:16]), stored)
|
||||||
|
if not matches or row is None:
|
||||||
|
raise unauthorized("Basic")
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
digest = self.token_hash(token)
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
db.execute("DELETE FROM sessions WHERE expires_at <= now()")
|
||||||
|
session = db.execute("""
|
||||||
|
INSERT INTO sessions (token_hash,user_id,expires_at)
|
||||||
|
VALUES (%s,%s,clock_timestamp() + %s * interval '1 second')
|
||||||
|
RETURNING expires_at
|
||||||
|
""", (digest, row["id"], self.ttl)).fetchone()
|
||||||
|
return {"token": token, "token_type": "Bearer", **session, "user": row}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def token_hash(token):
|
||||||
|
return hashlib.sha256(token.encode("ascii")).digest()
|
||||||
|
|
||||||
|
def actor(self, digest):
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
row = db.execute("""
|
||||||
|
SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()
|
||||||
|
""", (digest,)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise unauthorized("Bearer")
|
||||||
|
return row["user_id"]
|
||||||
|
|
||||||
|
def user(self, user_id):
|
||||||
|
key = f"user:v1:{user_id}"
|
||||||
|
value = self.cache.get(key)
|
||||||
|
if value is not None:
|
||||||
|
try:
|
||||||
|
return json.loads(value)
|
||||||
|
except ValueError:
|
||||||
|
LOG.info("invalid profile cache value")
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
user = db.execute(f"SELECT {USER} FROM users WHERE id=%s", (user_id,)).fetchone()
|
||||||
|
self.cache.set(key, json.dumps(user, default=json_value))
|
||||||
|
return user
|
||||||
|
|
||||||
|
def logout(self, digest):
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
removed = db.execute("""
|
||||||
|
DELETE FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()
|
||||||
|
""", (digest,)).rowcount
|
||||||
|
if not removed:
|
||||||
|
raise unauthorized("Bearer")
|
||||||
|
|
||||||
|
def create_chat(self, title, members):
|
||||||
|
try:
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
chat = db.execute("INSERT INTO chats (title) VALUES (%s) RETURNING *",
|
||||||
|
(title,)).fetchone()
|
||||||
|
with db.cursor() as cursor:
|
||||||
|
cursor.executemany("INSERT INTO members (chat_id,user_id,position) VALUES (%s,%s,%s)",
|
||||||
|
[(chat["id"], member, index) for index, member in enumerate(members)])
|
||||||
|
return {**chat, "member_ids": members}
|
||||||
|
except psycopg.errors.ForeignKeyViolation:
|
||||||
|
raise invalid("Unknown member") from None
|
||||||
|
|
||||||
|
def chat(self, actor, chat_id):
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
row = db.execute(f"""
|
||||||
|
SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id
|
||||||
|
WHERE m.user_id=%s AND c.id=%s
|
||||||
|
""", (actor, chat_id)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise not_found()
|
||||||
|
return row
|
||||||
|
|
||||||
|
def send(self, actor, chat_id, text):
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
return db.execute(f"""
|
||||||
|
INSERT INTO messages (chat_id,sender_id,text) VALUES (%s,%s,%s) RETURNING {MESSAGE}
|
||||||
|
""", (chat_id, actor, text)).fetchone()
|
||||||
|
|
||||||
|
def chat_page(self, actor, limit, cursor):
|
||||||
|
return self.page(f"""SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id
|
||||||
|
WHERE m.user_id=%s""", "c.", ("chats", actor), limit, cursor)
|
||||||
|
|
||||||
|
def message_page(self, chat_id, limit, cursor):
|
||||||
|
return self.page(f"SELECT {MESSAGE} FROM messages WHERE chat_id=%s",
|
||||||
|
"", ("messages", chat_id), limit, cursor)
|
||||||
|
|
||||||
|
def page(self, query, prefix, scope, limit, cursor):
|
||||||
|
columns = f"{prefix}created_at, {prefix}id"
|
||||||
|
params = [scope[1]]
|
||||||
|
if cursor is not None:
|
||||||
|
params.extend(self.decode_cursor(cursor, scope))
|
||||||
|
query += f" AND ({columns}) > (%s,%s)"
|
||||||
|
with self.pool.connection() as db:
|
||||||
|
rows = db.execute(f"{query} ORDER BY {columns} LIMIT %s", (*params, limit + 1)).fetchall()
|
||||||
|
items = rows[:limit]
|
||||||
|
next_cursor = self.encode_cursor(scope, items[-1]) if len(rows) > limit else None
|
||||||
|
return {"items": items, "next_cursor": next_cursor}
|
||||||
|
|
||||||
|
def encode_cursor(self, scope, row):
|
||||||
|
payload = json.dumps([scope, row["created_at"], row["id"]],
|
||||||
|
separators=(",", ":"), default=json_value).encode("ascii")
|
||||||
|
signature = hmac.digest(self.cursor_key, payload, "sha256")
|
||||||
|
return base64.urlsafe_b64encode(signature + payload).decode("ascii").rstrip("=")
|
||||||
|
|
||||||
|
def decode_cursor(self, cursor, scope):
|
||||||
|
try:
|
||||||
|
if not 1 <= len(cursor) <= 256 or not cursor.isascii():
|
||||||
|
raise ValueError
|
||||||
|
data = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)
|
||||||
|
signature, payload = data[:32], data[32:]
|
||||||
|
if not hmac.compare_digest(signature, hmac.digest(self.cursor_key, payload, "sha256")):
|
||||||
|
raise ValueError
|
||||||
|
saved_scope, timestamp, record = json.loads(payload)
|
||||||
|
if saved_scope != [scope[0], str(scope[1])]:
|
||||||
|
raise ValueError
|
||||||
|
return datetime.fromisoformat(timestamp), UUID(record)
|
||||||
|
except ValueError:
|
||||||
|
raise invalid("Invalid cursor") from None
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import re
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
class ApiError(Exception):
|
||||||
|
def __init__(self, status, code, message, headers=None):
|
||||||
|
super().__init__(message)
|
||||||
|
self.status = status
|
||||||
|
self.code = code
|
||||||
|
self.headers = headers or {}
|
||||||
|
|
||||||
|
|
||||||
|
def invalid(message):
|
||||||
|
return ApiError(400, "invalid_request", message)
|
||||||
|
|
||||||
|
|
||||||
|
def not_found():
|
||||||
|
return ApiError(404, "not_found", "Resource not found")
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_id(value):
|
||||||
|
if not isinstance(value, str) or not re.fullmatch(
|
||||||
|
r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value
|
||||||
|
):
|
||||||
|
raise invalid("Invalid UUID")
|
||||||
|
return UUID(value)
|
||||||
|
|
||||||
|
|
||||||
|
def require_fields(body, *fields):
|
||||||
|
if not isinstance(body, dict) or body.keys() != set(fields):
|
||||||
|
raise invalid("Invalid request fields")
|
||||||
|
|
||||||
|
|
||||||
|
def require_text(value, maximum):
|
||||||
|
if (
|
||||||
|
not isinstance(value, str)
|
||||||
|
or not value.strip()
|
||||||
|
or len(value) > maximum
|
||||||
|
or "\x00" in value
|
||||||
|
or re.search(r"[\ud800-\udfff]", value)
|
||||||
|
):
|
||||||
|
raise invalid("Invalid text field")
|
||||||
|
return value
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import base64
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||||
|
from client import Client, Failure, check
|
||||||
|
|
||||||
|
|
||||||
|
def compose(*args):
|
||||||
|
subprocess.run(["docker", "compose", *args], check=True, timeout=45,
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||||
|
|
||||||
|
|
||||||
|
def ready(client):
|
||||||
|
deadline = time.monotonic() + 30
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
client.call("GET", "/health/ready", schema="Health")
|
||||||
|
return
|
||||||
|
except Failure:
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise
|
||||||
|
time.sleep(0.25)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
client = Client(timeout=4)
|
||||||
|
actor = client.user("deps")
|
||||||
|
path = "/api/v1/users/me"
|
||||||
|
client.call("GET", path, actor=actor, schema="User")
|
||||||
|
deadline = time.monotonic() + 5
|
||||||
|
while True:
|
||||||
|
cached = subprocess.check_output(
|
||||||
|
["docker", "compose", "exec", "-T", "cache", "redis-cli",
|
||||||
|
"EXISTS", f"user:v1:{actor['id']}"],
|
||||||
|
text=True, timeout=5,
|
||||||
|
).strip()
|
||||||
|
if cached == "1":
|
||||||
|
break
|
||||||
|
check(time.monotonic() < deadline, "profile was not cached")
|
||||||
|
time.sleep(0.1)
|
||||||
|
print("PASS: Redis contains the profile entry", flush=True)
|
||||||
|
wrong = base64.b64encode((actor["user"]["username"] + ":wrong-password").encode()).decode()
|
||||||
|
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + wrong})
|
||||||
|
client.call("GET", path, 401, headers={"X-User-Id": actor["id"]})
|
||||||
|
body = {**actor["registration"], "username": "race_" + uuid.uuid4().hex[:16]}
|
||||||
|
|
||||||
|
def register(_):
|
||||||
|
return Client().call("POST", "/api/v1/users", [201, 409], body=body)["status"]
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||||
|
results = list(pool.map(register, range(4)))
|
||||||
|
check(sorted(results) == [201, 409, 409, 409], f"registration race: {results}")
|
||||||
|
before = client.call("GET", "/api/v1/chats", actor=actor, schema="ChatPage")["json"]
|
||||||
|
client.call("POST", "/api/v1/chats", 400, actor=actor, body={
|
||||||
|
"title": "Rollback", "member_ids": [actor["id"], str(uuid.uuid4())],
|
||||||
|
})
|
||||||
|
after = client.call("GET", "/api/v1/chats", actor=actor, schema="ChatPage")["json"]
|
||||||
|
check(before == after, "partial chat survived rollback")
|
||||||
|
print("PASS: wrong password, X-User-Id rejected, concurrent registration, chat rollback", flush=True)
|
||||||
|
|
||||||
|
for service, status in (("cache", 200), ("db", 503)):
|
||||||
|
compose("stop", service)
|
||||||
|
try:
|
||||||
|
client.call("GET", path, status, actor=actor, schema="User" if status == 200 else None)
|
||||||
|
client.call("GET", path, 401)
|
||||||
|
finally:
|
||||||
|
compose("start", service)
|
||||||
|
ready(client)
|
||||||
|
print(f"PASS: {service} stopped, authenticated GET returned {status}", flush=True)
|
||||||
|
|
||||||
|
client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor)
|
||||||
|
client.call("GET", path, 401, actor=actor)
|
||||||
|
|
||||||
|
env = dict(os.environ, SESSION_TTL_SECONDS="2")
|
||||||
|
command = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "api"]
|
||||||
|
subprocess.run(command, env=env, check=True, timeout=45, stdout=subprocess.DEVNULL)
|
||||||
|
try:
|
||||||
|
ready(client)
|
||||||
|
short_session = client.user("expiry")
|
||||||
|
client.call("GET", path, actor=short_session, schema="User")
|
||||||
|
time.sleep(2.2)
|
||||||
|
client.call("GET", path, 401, actor=short_session)
|
||||||
|
finally:
|
||||||
|
compose("up", "-d", "--no-deps", "--force-recreate", "api")
|
||||||
|
ready(client)
|
||||||
|
print("PASS: logout and 2-second session expiry")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
\set ON_ERROR_STOP on
|
||||||
|
BEGIN;
|
||||||
|
INSERT INTO users (username,display_name,password_hash)
|
||||||
|
VALUES ('plan_' || substring(gen_random_uuid()::text,1,8),'Plan',decode('00','hex'))
|
||||||
|
RETURNING id AS actor_id \gset
|
||||||
|
INSERT INTO chats (title) VALUES ('Plan') RETURNING id AS chat_id \gset
|
||||||
|
INSERT INTO members (chat_id,user_id,position) VALUES (:'chat_id',:'actor_id',0);
|
||||||
|
INSERT INTO messages (chat_id,sender_id,text,created_at)
|
||||||
|
SELECT :'chat_id', :'actor_id', 'Synthetic history', now() + n * interval '1 millisecond'
|
||||||
|
FROM generate_series(1,12000) AS n;
|
||||||
|
ANALYZE messages;
|
||||||
|
SELECT created_at AS cursor_at, id AS cursor_id
|
||||||
|
FROM messages WHERE chat_id=:'chat_id' ORDER BY created_at,id OFFSET 9000 LIMIT 1 \gset
|
||||||
|
EXPLAIN (ANALYZE, BUFFERS)
|
||||||
|
SELECT id,chat_id,sender_id,text,created_at FROM messages
|
||||||
|
WHERE chat_id=:'chat_id' AND (created_at,id) > (:'cursor_at'::timestamptz,:'cursor_id'::uuid)
|
||||||
|
ORDER BY created_at,id LIMIT 51;
|
||||||
|
ROLLBACK;
|
||||||
Reference in New Issue
Block a user