From 388f31c5f4b2f96a1620fa9b04c71fed9acff47d Mon Sep 17 00:00:00 2001 From: Krep4334 Date: Fri, 13 Nov 2026 18:22:26 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=BD=D1=83=D1=8E=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 1 + .github/pull_request_template.md | 25 +-- .github/workflows/contracts.yml | 28 ++- Dockerfile | 13 ++ REPORT.md | 62 +++--- compose.yaml | 59 ++++++ evidence/local-checks.txt | 34 ++++ migrations/001_initial.sql | 43 +++++ requirements.txt | 5 + scripts/restart.cmd | 3 + scripts/restart.sh | 4 + scripts/setup.py | 28 +++ src/config.py | 13 ++ src/migrate.py | 19 ++ src/profile_cache.py | 56 ++++++ src/server.py | 288 ++++++++++++++++++++++++++++ src/storage.py | 211 ++++++++++++++++++++ src/validation.py | 43 +++++ student-tests/check_dependencies.py | 96 ++++++++++ student-tests/explain.sql | 18 ++ 20 files changed, 1009 insertions(+), 40 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/local-checks.txt create mode 100644 migrations/001_initial.sql create mode 100644 requirements.txt create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 scripts/setup.py create mode 100644 src/config.py create mode 100644 src/migrate.py create mode 100644 src/profile_cache.py create mode 100644 src/server.py create mode 100644 src/storage.py create mode 100644 src/validation.py create mode 100644 student-tests/check_dependencies.py create mode 100644 student-tests/explain.sql diff --git a/.dockerignore b/.dockerignore index bc16e556..ee6acd96 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +**/__pycache__ diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8ceefa10..e0307b46 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,13 @@ -## Лабораторная 2 +## Лабораторная 2. PostgreSQL и сессии -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор и группа — Samodurov Nikita, bpi2303 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Целевая оценка — 4 -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Пользователи, чаты, участники, сообщения и сессии хранятся в PostgreSQL. Вход создаёт Bearer-сессию с TTL, logout отзывает её. Redis кэширует неизменяемый профиль. + +Команды запуска и ограничения описаны в [REPORT.md](REPORT.md). Результаты smoke, перезапуска и дополнительных сценариев сохранены в [evidence/local-checks.txt](evidence/local-checks.txt). + +Источники — материалы и контракт курса. + +Использование ИИ — консультации и проверки ИИ. diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289f..8e0196df 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,5 +12,29 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Build and start stack + run: | + python scripts/setup.py + chmod +x scripts/restart.sh + docker compose build migrate + docker compose up -d --wait + - name: Run HTTP and resilience checks + run: | + make test + make test-persistence ACTION_SCRIPT=scripts/restart.sh + python student-tests/check_dependencies.py + docker compose exec -T db psql -U messenger -d messenger < student-tests/explain.sql + - name: Show service status + if: failure() + run: docker compose ps -a + - name: Stop stack + if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..0b38d1e8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.13.5-slim-bookworm@sha256:4c2cf9917bd1cbacc5e9b07320025bdb7cdf2df7b0ceaccb55e9dd7e30987419 + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY migrations/ ./migrations/ +COPY src/ ./src/ +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=3s --timeout=3s --start-period=5s --retries=4 \ + CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=2).close()"] +CMD ["python", "src/server.py"] diff --git a/REPORT.md b/REPORT.md index e42f2b05..4f990dfe 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,53 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор и группа — Samodurov Nikita, bpi2303 +- Целевая оценка — 4 +- Источники — материалы и контракт курса. +- Использование ИИ — консультации и проверки ИИ. -## Запуск из чистого clone +## Запуск из чистого клона -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2 и Python 3.9+ на хосте. Контейнер использует Python 3.13.5. По умолчанию API доступен на http://127.0.0.1:8080. При занятом порте задайте `PUBLIC_PORT` и соответствующий `BASE_URL`. Команда setup создаёт локальные `.env` и `secrets/`. -## Архитектура и решения +```bash +python scripts/setup.py +make check +docker compose config --quiet +docker compose build migrate +docker compose up -d --wait +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +python student-tests/check_dependencies.py +docker compose exec -T db psql -U messenger -d messenger < student-tests/explain.sql +docker compose down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +В Windows PowerShell используйте следующие проверки после запуска стенда. Перед проверками установите `$env:PYTHONUTF8 = '1'`. -## Доказательства критериев +```powershell +python scripts/check_contract.py +python tests/smoke.py +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +python student-tests/check_dependencies.py +Get-Content student-tests/explain.sql -Raw | docker compose exec -T db psql -U messenger -d messenger +``` -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Для полной валидации контракта установите `requirements-dev.txt` в отдельное окружение и выполните `python scripts/validate_openapi.py`. Команда `docker compose down` оставляет именованные тома. Для намеренного сброса учебных данных выполните `docker compose down -v`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура -## Эксперименты и отказы +Из ЛР 1 перенесены HTTP-обработчик, проверки полей, логика доступа и курсорная пагинация. Стенд состоит из API, PostgreSQL и Redis. Пароли хешируются scrypt с индивидуальной солью. В PostgreSQL хранится SHA-256 от случайного 256-битного токена. Срок сессии проверяется по часам PostgreSQL. Заголовок X-User-Id не участвует в аутентификации. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Уникальность username обеспечивает индекс PostgreSQL. Чат и его участники записываются в одной транзакции. Внешние ключи не допускают участников вне списка пользователей и сообщения от пользователя вне чата. SQL использует параметры. Пул ограничен восемью соединениями на процесс, ожидание свободного соединения ограничено 1,5 с, выполнение SQL — 3 с. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +PostgreSQL остаётся источником истины. После проверки сессии запрос к /users/me может взять неизменяемый профиль из Redis с TTL 300 с. При задержке свыше 350 мс API читает профиль из PostgreSQL. Очередь кэша ограничена четырьмя операциями. Изменение профиля не входит в API этой работы. -## Ограничения и следующие шаги +История использует составной индекс (chat_id, created_at, id) и курсор по паре (created_at, id). Подпись курсора проверяется общим ключом из secrets. Между страницами возможны конкурентные вставки, поэтому обход не фиксирует снимок истории. Поздно закоммиченная запись с ключом до курсора видна при новом обходе. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Проверки + +Результаты локального Docker-стенда и команды приведены в [evidence/local-checks.txt](evidence/local-checks.txt). + +## Ограничения + +При остановке PostgreSQL защищённый API недоступен. При отключении Redis профили читаются из PostgreSQL. Рестарт сохраняет данные и активные сессии при сохранённом томе pgdata. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..dc8e5efd --- /dev/null +++ b/compose.yaml @@ -0,0 +1,59 @@ +x-app: &app + image: mtusi-messenger:lab2 + env_file: .env + environment: + CURSOR_KEY_FILE: /run/secrets/cursor_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url, cache_url, cursor_key] + +services: + db: + image: postgres:17.5-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + cache: + image: redis:7.4.3-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "32mb", "--maxmemory-policy", "allkeys-lru"] + + migrate: + <<: *app + build: . + command: ["python", "src/migrate.py"] + healthcheck: + disable: true + depends_on: + db: + condition: service_healthy + + api: + <<: *app + ports: ["127.0.0.1:${PUBLIC_PORT:-8080}:${APP_PORT:-8080}"] + depends_on: + migrate: + condition: service_completed_successfully + cache: + condition: service_started + stop_grace_period: 8s + +secrets: + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + cursor_key: + file: ./secrets/cursor_key + +volumes: + pgdata: diff --git a/evidence/local-checks.txt b/evidence/local-checks.txt new file mode 100644 index 00000000..656eb3b3 --- /dev/null +++ b/evidence/local-checks.txt @@ -0,0 +1,34 @@ +2026-11-12. Windows, Python 3.13.5, Docker Desktop с Linux-контейнерами. +PUBLIC_PORT=18082, BASE_URL=http://127.0.0.1:18082 + +> python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +> python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +> python tests/smoke.py --base-url http://127.0.0.1:18082 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. + +> python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +Hook: docker compose restart db cache api. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +> python student-tests/check_dependencies.py +PASS: Redis contains the profile entry +PASS: wrong password, X-User-Id rejected, concurrent registration, chat rollback +PASS: cache stopped, authenticated GET returned 200 +PASS: db stopped, authenticated GET returned 503 +PASS: logout and 2-second session expiry + +> Get-Content student-tests/explain.sql -Raw | docker compose exec -T db psql -U messenger -d messenger +INSERT 0 12000 +ANALYZE +Limit (actual time=0.024..0.033 rows=51 loops=1) + Buffers: shared hit=3 + -> Index Scan using messages_history_idx on messages (actual time=0.024..0.030 rows=51 loops=1) + Index Cond: (chat_id = … AND ROW(created_at, id) > ROW(…, …)) + Buffers: shared hit=3 +Planning Time: 0.147 ms +Execution Time: 0.126 ms +ROLLBACK diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..3da9a650 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE members ( + chat_id uuid REFERENCES chats(id), + user_id uuid REFERENCES users(id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX members_user_idx ON members (user_id, chat_id); +CREATE INDEX chats_order_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES members (chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4ded7be4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +typing-extensions==4.14.0 diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..407349bd --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +docker compose restart db cache api diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..d3951a5d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart db cache api diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100644 index 00000000..930a97ff --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,28 @@ +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +directory = root / "secrets" +directory.mkdir(mode=0o700, exist_ok=True) + + +def ensure(name, value): + path = directory / name + try: + with path.open("x", encoding="utf-8") as file: + file.write(value + "\n") + except FileExistsError: + pass + return path.read_text(encoding="utf-8").strip() + + +password = ensure("db_password", secrets.token_hex(24)) +ensure("database_url", f"postgresql://messenger:{password}@db:5432/messenger") +ensure("cache_url", "redis://cache:6379/0") +ensure("cursor_key", secrets.token_hex(32)) + +try: + with (root / ".env").open("xb") as file: + file.write((root / ".env.example").read_bytes()) +except FileExistsError: + pass diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..fecaff1e --- /dev/null +++ b/src/config.py @@ -0,0 +1,13 @@ +import os +from pathlib import Path + + +def secret(name): + return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip() + + +def positive_int(name, default): + value = int(os.getenv(name, default)) + if value < 1: + raise ValueError(f"{name} must be positive") + return value diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..1c8bbfd8 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,19 @@ +from pathlib import Path +import psycopg +from config import secret + + +def main(): + with psycopg.connect(secret("DATABASE_URL"), connect_timeout=3) as connection: + connection.execute("SELECT pg_advisory_xact_lock(230301)") + connection.execute("CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY)") + for path in sorted(Path("migrations").glob("*.sql")): + if connection.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone(): + continue + connection.execute(path.read_text(encoding="utf-8")) + connection.execute("INSERT INTO schema_migrations (name) VALUES (%s)", (path.name,)) + print(f"Applied {path.name}") + + +if __name__ == "__main__": + main() diff --git a/src/profile_cache.py b/src/profile_cache.py new file mode 100644 index 00000000..c07ae5b8 --- /dev/null +++ b/src/profile_cache.py @@ -0,0 +1,56 @@ +from concurrent.futures import Future, TimeoutError +from queue import Empty, Full, Queue +from threading import Event, Thread + +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +class ProfileCache: + def __init__(self, url): + self.client = redis.Redis.from_url( + url, socket_timeout=0.25, socket_connect_timeout=0.25, + max_connections=1, retry=Retry(NoBackoff(), 0), + ) + self.work = Queue(maxsize=4) + self.stopping = Event() + Thread(target=self.run, daemon=True, name="profile-cache").start() + + def close(self): + self.stopping.set() + + def run(self): + try: + while not self.stopping.is_set(): + try: + result, operation, args = self.work.get(timeout=0.25) + except Empty: + continue + if not result.set_running_or_notify_cancel(): + continue + try: + result.set_result(operation(*args)) + except redis.RedisError: + result.set_result(None) + finally: + self.client.close() + + def submit(self, operation, *args): + result = Future() + try: + self.work.put_nowait((result, operation, args)) + except Full: + result.set_result(None) + return result + + def get(self, key): + result = self.submit(self.client.get, key) + try: + return result.result(timeout=0.35) + except TimeoutError: + result.cancel() + return None + + def set(self, key, value): + self.submit(self.client.setex, key, 300, value) diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..12d16617 --- /dev/null +++ b/src/server.py @@ -0,0 +1,288 @@ +import base64 +import json +import logging +import os +import re +import signal +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, unquote, urlsplit +from uuid import uuid4 + +from storage import DB_UNAVAILABLE, Store, json_value, unauthorized, unavailable +from validation import ApiError, canonical_id, invalid, not_found, require_fields, require_text + +CHAT_PATH = re.compile(r"/api/v1/chats/([^/]+)(/messages)?") +MAX_JSON_BYTES = 64 * 1024 +LOG = logging.getLogger("messenger") + + +def unique_object(pairs): + result = dict(pairs) + if len(result) != len(pairs): + raise ValueError + return result + + +def page_query(query): + try: + params = parse_qs(query, keep_blank_values=True, max_num_fields=16) + except ValueError: + raise invalid("Invalid query") from None + limits = params.get("limit", ["50"]) + cursors = params.get("cursor", [None]) + if len(limits) != 1 or not re.fullmatch(r"[0-9]{1,3}", limits[0]): + raise invalid("Invalid limit") + limit = int(limits[0]) + if not 1 <= limit <= 100: + raise invalid("Invalid limit") + if len(cursors) != 1: + raise invalid("Invalid cursor") + return limit, cursors[0] + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.handle_api() + + def do_POST(self): + self.handle_api() + + def do_DELETE(self): + self.handle_api() + + def log_message(self, format, *args): + pass + + def log_request(self, code="-", size="-"): + LOG.info("request_id=%s status=%s", self.request_id, code) + + def send_error(self, code, message=None, explain=None): + self.request_id = uuid4().hex + self.response_headers = {} + self.send_json(code, {"error": { + "code": "invalid_request", + "message": "Invalid HTTP request", + "request_id": self.request_id, + }}) + + def handle_api(self): + self.request_id = uuid4().hex + self.response_headers = {} + try: + try: + status, payload = self.route() + except ApiError as error: + status, payload = self.error_payload(error) + except DB_UNAVAILABLE: + status, payload = self.error_payload(unavailable()) + except (TimeoutError, ConnectionError): + return + except Exception: + LOG.error("request_id=%s unexpected error", self.request_id) + status, payload = self.error_payload( + ApiError(500, "internal_error", "Internal server error") + ) + self.send_json(status, payload) + except (TimeoutError, ConnectionError): + return + + def error_payload(self, error): + self.response_headers = error.headers + return error.status, {"error": { + "code": error.code, + "message": str(error), + "request_id": self.request_id, + }} + + def send_json(self, status, payload): + data = b"" if status == 204 else json.dumps(payload, ensure_ascii=False, default=json_value).encode("utf-8") + self.send_response(status) + self.send_header("X-Request-Id", self.request_id) + if status != 204: + self.send_header("Content-Type", "application/json") + for name, value in self.response_headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(data) + + def read_json(self): + lengths = self.headers.get_all("Content-Length", []) + if ( + self.headers.get("Transfer-Encoding") + or len(lengths) != 1 + or not re.fullmatch(r"[0-9]{1,10}", lengths[0]) + or self.headers.get_content_type() != "application/json" + ): + raise invalid("Expected a JSON body") + length = int(lengths[0]) + if length > MAX_JSON_BYTES: + raise invalid("JSON body is too large") + try: + data = self.rfile.read(length) + if len(data) != length: + raise ValueError + return json.loads(data.decode("utf-8"), object_pairs_hook=unique_object) + except (ValueError, RecursionError): + raise invalid("Invalid JSON") from None + + def route(self): + try: + target = urlsplit(self.path) + except ValueError: + raise invalid("Invalid request path") from None + path, method = target.path, self.command + app = self.server.app + + if method == "GET" and path == "/health/live": + return 200, {"status": "ok"} + if method == "GET" and path == "/health/ready": + app.ready() + return 200, {"status": "ok"} + if method == "POST" and path == "/api/v1/users": + body = self.read_json() + require_fields(body, "username", "display_name", "password") + username = body["username"] + if not isinstance(username, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", username): + raise invalid("Invalid username") + password = body["password"] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise invalid("Invalid password") + try: + password.encode("utf-8") + except UnicodeError: + raise invalid("Invalid password") from None + return 201, app.register(username, require_text(body["display_name"], 100), password) + if method == "POST" and path == "/api/v1/auth/sessions": + return 201, app.login(*self.basic_credentials()) + + match = CHAT_PATH.fullmatch(path) + if path not in ("/api/v1/users/me", "/api/v1/chats", "/api/v1/auth/sessions/current") and not match: + raise not_found() + header = self.headers.get("Authorization", "") + bearer = re.fullmatch(r"(?i:Bearer) ([A-Za-z0-9_-]{43})", header) + if not bearer: + raise unauthorized("Bearer") + digest = app.token_hash(bearer[1]) + actor = app.actor(digest) + if path == "/api/v1/auth/sessions/current" and method == "DELETE": + app.logout(digest) + return 204, None + if path == "/api/v1/users/me" and method == "GET": + return 200, app.user(actor) + if path == "/api/v1/chats": + if method == "POST": + body = self.read_json() + require_fields(body, "title", "member_ids") + title = require_text(body["title"], 100) + members = body["member_ids"] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise invalid("Invalid member list") + members = [canonical_id(member) for member in members] + if actor not in members or len(set(members)) != len(members): + raise invalid("Invalid member list") + return 201, app.create_chat(title, members) + if method == "GET": + return 200, app.chat_page(actor, *page_query(target.query)) + if match: + chat_id = canonical_id(unquote(match[1])) + chat = app.chat(actor, chat_id) + if match[2]: + if method == "POST": + body = self.read_json() + require_fields(body, "text") + return 201, app.send(actor, chat_id, require_text(body["text"], 4000)) + if method == "GET": + return 200, app.message_page(chat_id, *page_query(target.query)) + elif method == "GET": + return 200, chat + raise not_found() + + def basic_credentials(self): + if self.headers.get("Content-Length", "0") != "0" or self.headers.get("Transfer-Encoding"): + raise invalid("Login does not accept a body") + match = re.fullmatch(r"(?i:Basic) ([A-Za-z0-9+/=]{1,800})", + self.headers.get("Authorization", "")) + if not match: + raise unauthorized("Basic") + try: + value = base64.b64decode(match[1], validate=True).decode("utf-8") + except ValueError: + raise unauthorized("Basic") from None + username, separator, password = value.partition(":") + if not separator or not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128: + raise unauthorized("Basic") + return username, password + + +class Server(ThreadingHTTPServer): + timeout = 0.25 + request_queue_size = 64 + + def __init__(self, address): + self.app = Store() + self.stopping = False + self.active = 0 + self.workers = threading.Condition() + super().__init__(address, Handler) + + def get_request(self): + connection, address = super().get_request() + connection.settimeout(4) + return connection, address + + def verify_request(self, request, client_address): + return not self.stopping + + def process_request(self, request, client_address): + with self.workers: + self.active += 1 + try: + super().process_request(request, client_address) + except Exception: + self.worker_done() + raise + + def process_request_thread(self, request, client_address): + try: + super().process_request_thread(request, client_address) + finally: + self.worker_done() + + def worker_done(self): + with self.workers: + self.active -= 1 + self.workers.notify_all() + + def handle_error(self, request, client_address): + LOG.error("HTTP connection handler failed") + + def stop(self, signum, frame): + self.stopping = True + + def run(self): + while not self.stopping: + self.handle_request() + self.server_close() + with self.workers: + self.workers.wait_for(lambda: self.active == 0, timeout=5) + self.app.close() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + host = os.getenv("APP_HOST", "0.0.0.0") + port = int(os.getenv("APP_PORT", "8080")) + environment = os.getenv("APP_ENV", "lab") + with Server((host, port)) as server: + signal.signal(signal.SIGTERM, server.stop) + signal.signal(signal.SIGINT, server.stop) + LOG.info("started host=%s port=%s env=%s", host, server.server_port, environment) + server.run() + LOG.info("stopped") + + +if __name__ == "__main__": + main() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..884f14ad --- /dev/null +++ b/src/storage.py @@ -0,0 +1,211 @@ +import base64 +import hashlib +import hmac +import json +import logging +import secrets +from datetime import datetime +from threading import BoundedSemaphore +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from config import positive_int, secret +from profile_cache import ProfileCache +from validation import ApiError, invalid, not_found + +USER = "id, username, display_name, created_at" +MESSAGE = "id, chat_id, sender_id, text, created_at" +CHAT = """c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY position) AS member_ids""" +DB_UNAVAILABLE = (psycopg.OperationalError, PoolTimeout, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable) +LOG = logging.getLogger("messenger") + + +def json_value(value): + if isinstance(value, datetime): + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + if isinstance(value, UUID): + return str(value) + raise TypeError(type(value).__name__) + + +def unavailable(): + return ApiError(503, "unavailable", "Dependency temporarily unavailable") + + +def unauthorized(scheme): + challenge = f'{scheme} realm="messenger"' + if scheme == "Basic": + challenge += ', charset="UTF-8"' + return ApiError(401, "unauthorized", "Invalid credentials", {"WWW-Authenticate": challenge}) + + +class Store: + def __init__(self): + self.pool = ConnectionPool( + secret("DATABASE_URL"), min_size=1, max_size=8, max_waiting=24, timeout=1.5, + kwargs={"row_factory": dict_row, "connect_timeout": 2, + "options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"}, + open=True, check=ConnectionPool.check_connection, + ) + self.cache = ProfileCache(secret("CACHE_URL")) + self.cursor_key = bytes.fromhex(secret("CURSOR_KEY")) + self.ttl = positive_int("SESSION_TTL_SECONDS", 3600) + self.hash_slots = BoundedSemaphore(4) + self.dummy_hash = bytes(16 + 32) + + def close(self): + self.cache.close() + self.pool.close() + + def ready(self): + with self.pool.connection() as db: + db.execute("SELECT 1 FROM users LIMIT 1") + + def password_hash(self, password, salt): + if not self.hash_slots.acquire(timeout=0.5): + raise unavailable() + try: + return salt + hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=2**15, r=8, p=1, + maxmem=64 * 1024**2, dklen=32, + ) + finally: + self.hash_slots.release() + + def register(self, username, name, password): + password_hash = self.password_hash(password, secrets.token_bytes(16)) + try: + with self.pool.connection() as db: + return db.execute(f""" + INSERT INTO users (username,display_name,password_hash) VALUES (%s,%s,%s) + RETURNING {USER} + """, (username, name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Username already exists") from None + + def login(self, username, password): + with self.pool.connection() as db: + row = db.execute(f"SELECT {USER},password_hash FROM users WHERE username=%s", + (username,)).fetchone() + stored = row.pop("password_hash") if row else self.dummy_hash + matches = hmac.compare_digest(self.password_hash(password, stored[:16]), stored) + if not matches or row is None: + raise unauthorized("Basic") + token = secrets.token_urlsafe(32) + digest = self.token_hash(token) + with self.pool.connection() as db: + db.execute("DELETE FROM sessions WHERE expires_at <= now()") + session = db.execute(""" + INSERT INTO sessions (token_hash,user_id,expires_at) + VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') + RETURNING expires_at + """, (digest, row["id"], self.ttl)).fetchone() + return {"token": token, "token_type": "Bearer", **session, "user": row} + + @staticmethod + def token_hash(token): + return hashlib.sha256(token.encode("ascii")).digest() + + def actor(self, digest): + with self.pool.connection() as db: + row = db.execute(""" + SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp() + """, (digest,)).fetchone() + if row is None: + raise unauthorized("Bearer") + return row["user_id"] + + def user(self, user_id): + key = f"user:v1:{user_id}" + value = self.cache.get(key) + if value is not None: + try: + return json.loads(value) + except ValueError: + LOG.info("invalid profile cache value") + with self.pool.connection() as db: + user = db.execute(f"SELECT {USER} FROM users WHERE id=%s", (user_id,)).fetchone() + self.cache.set(key, json.dumps(user, default=json_value)) + return user + + def logout(self, digest): + with self.pool.connection() as db: + removed = db.execute(""" + DELETE FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp() + """, (digest,)).rowcount + if not removed: + raise unauthorized("Bearer") + + def create_chat(self, title, members): + try: + with self.pool.connection() as db: + chat = db.execute("INSERT INTO chats (title) VALUES (%s) RETURNING *", + (title,)).fetchone() + with db.cursor() as cursor: + cursor.executemany("INSERT INTO members (chat_id,user_id,position) VALUES (%s,%s,%s)", + [(chat["id"], member, index) for index, member in enumerate(members)]) + return {**chat, "member_ids": members} + except psycopg.errors.ForeignKeyViolation: + raise invalid("Unknown member") from None + + def chat(self, actor, chat_id): + with self.pool.connection() as db: + row = db.execute(f""" + SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id + WHERE m.user_id=%s AND c.id=%s + """, (actor, chat_id)).fetchone() + if row is None: + raise not_found() + return row + + def send(self, actor, chat_id, text): + with self.pool.connection() as db: + return db.execute(f""" + INSERT INTO messages (chat_id,sender_id,text) VALUES (%s,%s,%s) RETURNING {MESSAGE} + """, (chat_id, actor, text)).fetchone() + + def chat_page(self, actor, limit, cursor): + return self.page(f"""SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id + WHERE m.user_id=%s""", "c.", ("chats", actor), limit, cursor) + + def message_page(self, chat_id, limit, cursor): + return self.page(f"SELECT {MESSAGE} FROM messages WHERE chat_id=%s", + "", ("messages", chat_id), limit, cursor) + + def page(self, query, prefix, scope, limit, cursor): + columns = f"{prefix}created_at, {prefix}id" + params = [scope[1]] + if cursor is not None: + params.extend(self.decode_cursor(cursor, scope)) + query += f" AND ({columns}) > (%s,%s)" + with self.pool.connection() as db: + rows = db.execute(f"{query} ORDER BY {columns} LIMIT %s", (*params, limit + 1)).fetchall() + items = rows[:limit] + next_cursor = self.encode_cursor(scope, items[-1]) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} + + def encode_cursor(self, scope, row): + payload = json.dumps([scope, row["created_at"], row["id"]], + separators=(",", ":"), default=json_value).encode("ascii") + signature = hmac.digest(self.cursor_key, payload, "sha256") + return base64.urlsafe_b64encode(signature + payload).decode("ascii").rstrip("=") + + def decode_cursor(self, cursor, scope): + try: + if not 1 <= len(cursor) <= 256 or not cursor.isascii(): + raise ValueError + data = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + signature, payload = data[:32], data[32:] + if not hmac.compare_digest(signature, hmac.digest(self.cursor_key, payload, "sha256")): + raise ValueError + saved_scope, timestamp, record = json.loads(payload) + if saved_scope != [scope[0], str(scope[1])]: + raise ValueError + return datetime.fromisoformat(timestamp), UUID(record) + except ValueError: + raise invalid("Invalid cursor") from None diff --git a/src/validation.py b/src/validation.py new file mode 100644 index 00000000..66b881aa --- /dev/null +++ b/src/validation.py @@ -0,0 +1,43 @@ +import re +from uuid import UUID + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status = status + self.code = code + self.headers = headers or {} + + +def invalid(message): + return ApiError(400, "invalid_request", message) + + +def not_found(): + return ApiError(404, "not_found", "Resource not found") + + +def canonical_id(value): + if not isinstance(value, str) or not re.fullmatch( + r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value + ): + raise invalid("Invalid UUID") + return UUID(value) + + +def require_fields(body, *fields): + if not isinstance(body, dict) or body.keys() != set(fields): + raise invalid("Invalid request fields") + + +def require_text(value, maximum): + if ( + not isinstance(value, str) + or not value.strip() + or len(value) > maximum + or "\x00" in value + or re.search(r"[\ud800-\udfff]", value) + ): + raise invalid("Invalid text field") + return value diff --git a/student-tests/check_dependencies.py b/student-tests/check_dependencies.py new file mode 100644 index 00000000..f1ce8e76 --- /dev/null +++ b/student-tests/check_dependencies.py @@ -0,0 +1,96 @@ +import base64 +from concurrent.futures import ThreadPoolExecutor +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, check + + +def compose(*args): + subprocess.run(["docker", "compose", *args], check=True, timeout=45, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def ready(client): + deadline = time.monotonic() + 30 + while True: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + if time.monotonic() >= deadline: + raise + time.sleep(0.25) + + +def main(): + client = Client(timeout=4) + actor = client.user("deps") + path = "/api/v1/users/me" + client.call("GET", path, actor=actor, schema="User") + deadline = time.monotonic() + 5 + while True: + cached = subprocess.check_output( + ["docker", "compose", "exec", "-T", "cache", "redis-cli", + "EXISTS", f"user:v1:{actor['id']}"], + text=True, timeout=5, + ).strip() + if cached == "1": + break + check(time.monotonic() < deadline, "profile was not cached") + time.sleep(0.1) + print("PASS: Redis contains the profile entry", flush=True) + wrong = base64.b64encode((actor["user"]["username"] + ":wrong-password").encode()).decode() + client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + wrong}) + client.call("GET", path, 401, headers={"X-User-Id": actor["id"]}) + body = {**actor["registration"], "username": "race_" + uuid.uuid4().hex[:16]} + + def register(_): + return Client().call("POST", "/api/v1/users", [201, 409], body=body)["status"] + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(register, range(4))) + check(sorted(results) == [201, 409, 409, 409], f"registration race: {results}") + before = client.call("GET", "/api/v1/chats", actor=actor, schema="ChatPage")["json"] + client.call("POST", "/api/v1/chats", 400, actor=actor, body={ + "title": "Rollback", "member_ids": [actor["id"], str(uuid.uuid4())], + }) + after = client.call("GET", "/api/v1/chats", actor=actor, schema="ChatPage")["json"] + check(before == after, "partial chat survived rollback") + print("PASS: wrong password, X-User-Id rejected, concurrent registration, chat rollback", flush=True) + + for service, status in (("cache", 200), ("db", 503)): + compose("stop", service) + try: + client.call("GET", path, status, actor=actor, schema="User" if status == 200 else None) + client.call("GET", path, 401) + finally: + compose("start", service) + ready(client) + print(f"PASS: {service} stopped, authenticated GET returned {status}", flush=True) + + client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor) + client.call("GET", path, 401, actor=actor) + + env = dict(os.environ, SESSION_TTL_SECONDS="2") + command = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "api"] + subprocess.run(command, env=env, check=True, timeout=45, stdout=subprocess.DEVNULL) + try: + ready(client) + short_session = client.user("expiry") + client.call("GET", path, actor=short_session, schema="User") + time.sleep(2.2) + client.call("GET", path, 401, actor=short_session) + finally: + compose("up", "-d", "--no-deps", "--force-recreate", "api") + ready(client) + print("PASS: logout and 2-second session expiry") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 00000000..ffd4a5c8 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,18 @@ +\set ON_ERROR_STOP on +BEGIN; +INSERT INTO users (username,display_name,password_hash) +VALUES ('plan_' || substring(gen_random_uuid()::text,1,8),'Plan',decode('00','hex')) +RETURNING id AS actor_id \gset +INSERT INTO chats (title) VALUES ('Plan') RETURNING id AS chat_id \gset +INSERT INTO members (chat_id,user_id,position) VALUES (:'chat_id',:'actor_id',0); +INSERT INTO messages (chat_id,sender_id,text,created_at) +SELECT :'chat_id', :'actor_id', 'Synthetic history', now() + n * interval '1 millisecond' +FROM generate_series(1,12000) AS n; +ANALYZE messages; +SELECT created_at AS cursor_at, id AS cursor_id +FROM messages WHERE chat_id=:'chat_id' ORDER BY created_at,id OFFSET 9000 LIMIT 1 \gset +EXPLAIN (ANALYZE, BUFFERS) +SELECT id,chat_id,sender_id,text,created_at FROM messages +WHERE chat_id=:'chat_id' AND (created_at,id) > (:'cursor_at'::timestamptz,:'cursor_id'::uuid) +ORDER BY created_at,id LIMIT 51; +ROLLBACK; -- 2.54.0