From 2a4242695fefad0363ee4eb08b093904dbc86c32 Mon Sep 17 00:00:00 2001 From: Krep4334 Date: Fri, 13 Nov 2026 18:22:52 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=BD=D1=83=D1=8E=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 1 + .github/pull_request_template.md | 25 ++- .github/workflows/contracts.yml | 29 ++- Dockerfile | 13 ++ REPORT.md | 62 ++++--- compose.yaml | 85 +++++++++ evidence/local-checks.txt | 30 ++++ infra/haproxy.cfg | 34 ++++ migrations/001_initial.sql | 43 +++++ migrations/002_idempotency.sql | 11 ++ requirements.txt | 5 + scripts/restart.cmd | 3 + scripts/restart.sh | 4 + scripts/setup.py | 28 +++ scripts/stop-one.cmd | 7 + scripts/stop-one.sh | 7 + src/config.py | 13 ++ src/migrate.py | 19 ++ src/profile_cache.py | 56 ++++++ src/server.py | 293 +++++++++++++++++++++++++++++++ src/storage.py | 232 ++++++++++++++++++++++++ src/validation.py | 43 +++++ student-tests/check_replicas.py | 118 +++++++++++++ 23 files changed, 1121 insertions(+), 40 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/local-checks.txt create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 requirements.txt create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 scripts/setup.py create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh create mode 100644 src/config.py create mode 100644 src/migrate.py create mode 100644 src/profile_cache.py create mode 100644 src/server.py create mode 100644 src/storage.py create mode 100644 src/validation.py create mode 100644 student-tests/check_replicas.py diff --git a/.dockerignore b/.dockerignore index bc16e556..ee6acd96 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +**/__pycache__ diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e6e63eb2..51c833b8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,13 @@ -## Лабораторная 3 +## Лабораторная 3. Балансировка и идемпотентность -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор и группа — Samodurov Nikita, bpi2303 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Целевая оценка — 4 -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Две реплики API принимают запросы через HAProxy и используют общие сессии в PostgreSQL. Параллельные повторы отправки с одним ключом создают одно сообщение. + +Команды запуска и ограничения описаны в [REPORT.md](REPORT.md). Результаты smoke, перезапуска и дополнительных сценариев сохранены в [evidence/local-checks.txt](evidence/local-checks.txt). + +Источники — материалы и контракт курса. + +Использование ИИ — консультации и проверки ИИ. diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289f..28a2c994 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,5 +12,30 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Build and start stack + run: | + python scripts/setup.py + chmod +x scripts/restart.sh scripts/stop-one.sh + docker compose build migrate + docker compose up -d --wait + - name: Run HTTP and resilience checks + run: | + make test + make test-persistence ACTION_SCRIPT=scripts/restart.sh + make test-failover ACTION_SCRIPT=scripts/stop-one.sh + docker compose start api-a api-b + python student-tests/check_replicas.py + - name: Show service status + if: failure() + run: docker compose ps -a + - name: Stop stack + if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..0b38d1e8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.13.5-slim-bookworm@sha256:4c2cf9917bd1cbacc5e9b07320025bdb7cdf2df7b0ceaccb55e9dd7e30987419 + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY migrations/ ./migrations/ +COPY src/ ./src/ +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=3s --timeout=3s --start-period=5s --retries=4 \ + CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=2).close()"] +CMD ["python", "src/server.py"] diff --git a/REPORT.md b/REPORT.md index c2679005..cc54d8df 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,53 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор и группа — Samodurov Nikita, bpi2303 +- Целевая оценка — 4 +- Источники — материалы и контракт курса. +- Использование ИИ — консультации и проверки ИИ. -## Запуск из чистого clone +## Запуск из чистого клона -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2 и Python 3.9+ на хосте. Контейнер использует Python 3.13.5. По умолчанию API доступен на http://127.0.0.1:8080. При занятом порте задайте `PUBLIC_PORT` и соответствующий `BASE_URL`. Команда setup создаёт локальные `.env` и `secrets/`. -## Архитектура и решения +```bash +python scripts/setup.py +make check +docker compose config --quiet +docker compose build migrate +docker compose up -d --wait +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose start api-a api-b +python student-tests/check_replicas.py +docker compose down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +В Windows PowerShell используйте следующие проверки после запуска стенда. Перед проверками установите `$env:PYTHONUTF8 = '1'`. -## Доказательства критериев +```powershell +python scripts/check_contract.py +python tests/smoke.py +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +docker compose start api-a api-b +python student-tests/check_replicas.py +``` -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Для полной валидации контракта установите `requirements-dev.txt` в отдельное окружение и выполните `python scripts/validate_openapi.py`. Команда `docker compose down` оставляет именованные тома. Для намеренного сброса учебных данных выполните `docker compose down -v`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура -## Эксперименты и отказы +Из ЛР 2 перенесены PostgreSQL, сессии и кэш профилей. HAProxy распределяет запросы между api-a и api-b по round-robin и проверяет /health/ready раз в секунду. После двух неуспешных проверок реплика исключается. Таймаут каждой проверки — одна секунда. Штатный выход по SIGTERM закрывает слушающий сокет и оставляет обработчикам до пяти секунд. На хост опубликован только HAProxy. Алиас X-Instance-Id соответствует имени сервиса. Разрешение имён использует Docker DNS. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +PostgreSQL хранит результат отправки с ключом в области «пользователь + чат + POST». Транзакционная advisory-блокировка сериализует запросы с одинаковым ключом. Сообщение и запись результата фиксируются одним COMMIT. Тот же текст в течение 24 часов возвращает исходный ответ, другой текст даёт 409. После истечения TTL запись того же ключа заменяется при следующем использовании. Сессия и членство проверяются на каждом повторе. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Версионируемые миграции выполняет сервис migrate под блокировкой. Проверка замены реплик использует ту же версию схемы. Миграции с несовместимыми изменениями требуют отдельного плана обновления. -## Ограничения и следующие шаги +## Проверки -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Результаты локального Docker-стенда и команды приведены в [evidence/local-checks.txt](evidence/local-checks.txt). + +## Ограничения + +Реплики защищают от отказа одного процесса API. HAProxy и PostgreSQL работают по одному экземпляру. Во время исключения остановленного backend возможны короткие ошибки, POST сообщения можно повторить с исходным ключом. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..efd72d51 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,85 @@ +x-app: &app + image: mtusi-messenger:lab3 + cpus: 1.0 + mem_limit: 320m + env_file: .env + environment: + CURSOR_KEY_FILE: /run/secrets/cursor_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url, cache_url, cursor_key] + +services: + db: + cpus: 1.0 + mem_limit: 384m + image: postgres:17.5-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + cache: + cpus: 0.25 + mem_limit: 96m + image: redis:7.4.3-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "32mb", "--maxmemory-policy", "allkeys-lru"] + + migrate: + <<: *app + build: . + command: ["python", "src/migrate.py"] + healthcheck: + disable: true + depends_on: + db: + condition: service_healthy + + api-a: &api + <<: *app + environment: + CURSOR_KEY_FILE: /run/secrets/cursor_key + INSTANCE_ID: api-a + depends_on: + migrate: + condition: service_completed_successfully + cache: + condition: service_started + stop_grace_period: 8s + + api-b: + <<: *api + environment: + CURSOR_KEY_FILE: /run/secrets/cursor_key + INSTANCE_ID: api-b + + lb: + image: haproxy:3.0.11-alpine + cpus: 0.5 + mem_limit: 96m + ports: ["127.0.0.1:${PUBLIC_PORT:-8080}:8080"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: + api-a: + condition: service_healthy + api-b: + condition: service_healthy + +secrets: + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + cursor_key: + file: ./secrets/cursor_key + +volumes: + pgdata: diff --git a/evidence/local-checks.txt b/evidence/local-checks.txt new file mode 100644 index 00000000..67e18cf1 --- /dev/null +++ b/evidence/local-checks.txt @@ -0,0 +1,30 @@ +2026-11-12. Windows, Python 3.13.5, Docker Desktop с Linux-контейнерами. +PUBLIC_PORT=18083, BASE_URL=http://127.0.0.1:18083 + +> python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +> python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +> python tests/smoke.py --base-url http://127.0.0.1:18083 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. + +> python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +Hook: docker compose restart db cache api-a api-b lb. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +> python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +{"observed_instances_before": 2, "successful_writes": 106, "failed_attempts": 4, "first_recovery_seconds": 1.658} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +> python student-tests/check_replicas.py +PASS: 8 concurrent replays through both replicas, 1 message +PASS: stop api-a, 5.8 write/s, errors=3, survivor=0.085s +PASS: stop api-b, 3.4 write/s, errors=5, survivor=0.048s +PASS: recreate api-a, 8.2 write/s, errors=2, survivor=0.130s +PASS: recreate api-b, 8.0 write/s, errors=3, survivor=0.094s + +write/s считается за весь короткий сценарий. survivor — время до первого +успешного ответа оставшейся реплики после начала действия. Публичный failover +отдельно измеряет первый ответ после завершения hook. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..fa9f06b6 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,34 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + option http-server-close + timeout connect 1s + timeout client 15s + timeout server 12s + timeout http-request 8s + timeout check 1s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 500ms + timeout retry 500ms + hold valid 1s + hold nx 1s + +frontend messenger + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr libc,none + server api-a api-a:8080 + server api-b api-b:8080 diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..3da9a650 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE members ( + chat_id uuid REFERENCES chats(id), + user_id uuid REFERENCES users(id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX members_user_idx ON members (user_id, chat_id); +CREATE INDEX chats_order_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES members (chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..01d78cb9 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_requests ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL, + fingerprint bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (actor_id, chat_id, key) +); + +CREATE INDEX message_requests_expiry_idx ON message_requests (expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4ded7be4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +typing-extensions==4.14.0 diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..de664b26 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +docker compose restart db cache api-a api-b lb diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..9f2c80d3 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart db cache api-a api-b lb diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100644 index 00000000..930a97ff --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,28 @@ +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +directory = root / "secrets" +directory.mkdir(mode=0o700, exist_ok=True) + + +def ensure(name, value): + path = directory / name + try: + with path.open("x", encoding="utf-8") as file: + file.write(value + "\n") + except FileExistsError: + pass + return path.read_text(encoding="utf-8").strip() + + +password = ensure("db_password", secrets.token_hex(24)) +ensure("database_url", f"postgresql://messenger:{password}@db:5432/messenger") +ensure("cache_url", "redis://cache:6379/0") +ensure("cursor_key", secrets.token_hex(32)) + +try: + with (root / ".env").open("xb") as file: + file.write((root / ".env.example").read_bytes()) +except FileExistsError: + pass diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..c7718fe6 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,7 @@ +@echo off +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="api-a" goto stop +if "%TARGET_INSTANCE_ID%"=="api-b" goto stop +exit /b 2 +:stop +docker compose stop --timeout 5 %TARGET_INSTANCE_ID% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..1f987b89 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +case "${TARGET_INSTANCE_ID:-}" in + api-a|api-b) docker compose stop --timeout 5 "$TARGET_INSTANCE_ID" ;; + *) exit 2 ;; +esac diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..fecaff1e --- /dev/null +++ b/src/config.py @@ -0,0 +1,13 @@ +import os +from pathlib import Path + + +def secret(name): + return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip() + + +def positive_int(name, default): + value = int(os.getenv(name, default)) + if value < 1: + raise ValueError(f"{name} must be positive") + return value diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..1c8bbfd8 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,19 @@ +from pathlib import Path +import psycopg +from config import secret + + +def main(): + with psycopg.connect(secret("DATABASE_URL"), connect_timeout=3) as connection: + connection.execute("SELECT pg_advisory_xact_lock(230301)") + connection.execute("CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY)") + for path in sorted(Path("migrations").glob("*.sql")): + if connection.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone(): + continue + connection.execute(path.read_text(encoding="utf-8")) + connection.execute("INSERT INTO schema_migrations (name) VALUES (%s)", (path.name,)) + print(f"Applied {path.name}") + + +if __name__ == "__main__": + main() diff --git a/src/profile_cache.py b/src/profile_cache.py new file mode 100644 index 00000000..c07ae5b8 --- /dev/null +++ b/src/profile_cache.py @@ -0,0 +1,56 @@ +from concurrent.futures import Future, TimeoutError +from queue import Empty, Full, Queue +from threading import Event, Thread + +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +class ProfileCache: + def __init__(self, url): + self.client = redis.Redis.from_url( + url, socket_timeout=0.25, socket_connect_timeout=0.25, + max_connections=1, retry=Retry(NoBackoff(), 0), + ) + self.work = Queue(maxsize=4) + self.stopping = Event() + Thread(target=self.run, daemon=True, name="profile-cache").start() + + def close(self): + self.stopping.set() + + def run(self): + try: + while not self.stopping.is_set(): + try: + result, operation, args = self.work.get(timeout=0.25) + except Empty: + continue + if not result.set_running_or_notify_cancel(): + continue + try: + result.set_result(operation(*args)) + except redis.RedisError: + result.set_result(None) + finally: + self.client.close() + + def submit(self, operation, *args): + result = Future() + try: + self.work.put_nowait((result, operation, args)) + except Full: + result.set_result(None) + return result + + def get(self, key): + result = self.submit(self.client.get, key) + try: + return result.result(timeout=0.35) + except TimeoutError: + result.cancel() + return None + + def set(self, key, value): + self.submit(self.client.setex, key, 300, value) diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..53b4e1ea --- /dev/null +++ b/src/server.py @@ -0,0 +1,293 @@ +import base64 +import json +import logging +import os +import re +import signal +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, unquote, urlsplit +from uuid import uuid4 + +from storage import DB_UNAVAILABLE, Store, json_value, unauthorized, unavailable +from validation import ApiError, canonical_id, invalid, not_found, require_fields, require_text + +CHAT_PATH = re.compile(r"/api/v1/chats/([^/]+)(/messages)?") +MAX_JSON_BYTES = 64 * 1024 +LOG = logging.getLogger("messenger") + + +def unique_object(pairs): + result = dict(pairs) + if len(result) != len(pairs): + raise ValueError + return result + + +def page_query(query): + try: + params = parse_qs(query, keep_blank_values=True, max_num_fields=16) + except ValueError: + raise invalid("Invalid query") from None + limits = params.get("limit", ["50"]) + cursors = params.get("cursor", [None]) + if len(limits) != 1 or not re.fullmatch(r"[0-9]{1,3}", limits[0]): + raise invalid("Invalid limit") + limit = int(limits[0]) + if not 1 <= limit <= 100: + raise invalid("Invalid limit") + if len(cursors) != 1: + raise invalid("Invalid cursor") + return limit, cursors[0] + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.handle_api() + + def do_POST(self): + self.handle_api() + + def do_DELETE(self): + self.handle_api() + + def log_message(self, format, *args): + pass + + def log_request(self, code="-", size="-"): + LOG.info("request_id=%s status=%s", self.request_id, code) + + def send_error(self, code, message=None, explain=None): + self.request_id = uuid4().hex + self.response_headers = {} + self.send_json(code, {"error": { + "code": "invalid_request", + "message": "Invalid HTTP request", + "request_id": self.request_id, + }}) + + def handle_api(self): + self.request_id = uuid4().hex + self.response_headers = {} + try: + try: + status, payload = self.route() + except ApiError as error: + status, payload = self.error_payload(error) + except DB_UNAVAILABLE: + status, payload = self.error_payload(unavailable()) + except (TimeoutError, ConnectionError): + return + except Exception: + LOG.error("request_id=%s unexpected error", self.request_id) + status, payload = self.error_payload( + ApiError(500, "internal_error", "Internal server error") + ) + self.send_json(status, payload) + except (TimeoutError, ConnectionError): + return + + def error_payload(self, error): + self.response_headers = error.headers + return error.status, {"error": { + "code": error.code, + "message": str(error), + "request_id": self.request_id, + }} + + def send_json(self, status, payload): + data = b"" if status == 204 else json.dumps(payload, ensure_ascii=False, default=json_value).encode("utf-8") + self.send_response(status) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.server.instance_id) + if status != 204: + self.send_header("Content-Type", "application/json") + for name, value in self.response_headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(data) + + def read_json(self): + lengths = self.headers.get_all("Content-Length", []) + if ( + self.headers.get("Transfer-Encoding") + or len(lengths) != 1 + or not re.fullmatch(r"[0-9]{1,10}", lengths[0]) + or self.headers.get_content_type() != "application/json" + ): + raise invalid("Expected a JSON body") + length = int(lengths[0]) + if length > MAX_JSON_BYTES: + raise invalid("JSON body is too large") + try: + data = self.rfile.read(length) + if len(data) != length: + raise ValueError + return json.loads(data.decode("utf-8"), object_pairs_hook=unique_object) + except (ValueError, RecursionError): + raise invalid("Invalid JSON") from None + + def route(self): + try: + target = urlsplit(self.path) + except ValueError: + raise invalid("Invalid request path") from None + path, method = target.path, self.command + app = self.server.app + + if method == "GET" and path == "/health/live": + return 200, {"status": "ok"} + if method == "GET" and path == "/health/ready": + app.ready() + return 200, {"status": "ok"} + if method == "POST" and path == "/api/v1/users": + body = self.read_json() + require_fields(body, "username", "display_name", "password") + username = body["username"] + if not isinstance(username, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", username): + raise invalid("Invalid username") + password = body["password"] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise invalid("Invalid password") + try: + password.encode("utf-8") + except UnicodeError: + raise invalid("Invalid password") from None + return 201, app.register(username, require_text(body["display_name"], 100), password) + if method == "POST" and path == "/api/v1/auth/sessions": + return 201, app.login(*self.basic_credentials()) + + match = CHAT_PATH.fullmatch(path) + if path not in ("/api/v1/users/me", "/api/v1/chats", "/api/v1/auth/sessions/current") and not match: + raise not_found() + header = self.headers.get("Authorization", "") + bearer = re.fullmatch(r"(?i:Bearer) ([A-Za-z0-9_-]{43})", header) + if not bearer: + raise unauthorized("Bearer") + digest = app.token_hash(bearer[1]) + actor = app.actor(digest) + if path == "/api/v1/auth/sessions/current" and method == "DELETE": + app.logout(digest) + return 204, None + if path == "/api/v1/users/me" and method == "GET": + return 200, app.user(actor) + if path == "/api/v1/chats": + if method == "POST": + body = self.read_json() + require_fields(body, "title", "member_ids") + title = require_text(body["title"], 100) + members = body["member_ids"] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise invalid("Invalid member list") + members = [canonical_id(member) for member in members] + if actor not in members or len(set(members)) != len(members): + raise invalid("Invalid member list") + return 201, app.create_chat(title, members) + if method == "GET": + return 200, app.chat_page(actor, *page_query(target.query)) + if match: + chat_id = canonical_id(unquote(match[1])) + chat = app.chat(actor, chat_id) + if match[2]: + if method == "POST": + body = self.read_json() + require_fields(body, "text") + key = self.headers.get("Idempotency-Key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise invalid("Invalid Idempotency-Key") + return 201, app.send(actor, chat_id, require_text(body["text"], 4000), key) + if method == "GET": + return 200, app.message_page(chat_id, *page_query(target.query)) + elif method == "GET": + return 200, chat + raise not_found() + + def basic_credentials(self): + if self.headers.get("Content-Length", "0") != "0" or self.headers.get("Transfer-Encoding"): + raise invalid("Login does not accept a body") + match = re.fullmatch(r"(?i:Basic) ([A-Za-z0-9+/=]{1,800})", + self.headers.get("Authorization", "")) + if not match: + raise unauthorized("Basic") + try: + value = base64.b64decode(match[1], validate=True).decode("utf-8") + except ValueError: + raise unauthorized("Basic") from None + username, separator, password = value.partition(":") + if not separator or not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128: + raise unauthorized("Basic") + return username, password + + +class Server(ThreadingHTTPServer): + timeout = 0.25 + request_queue_size = 64 + + def __init__(self, address): + self.app = Store() + self.instance_id = os.getenv("INSTANCE_ID") or uuid4().hex + self.stopping = False + self.active = 0 + self.workers = threading.Condition() + super().__init__(address, Handler) + + def get_request(self): + connection, address = super().get_request() + connection.settimeout(4) + return connection, address + + def verify_request(self, request, client_address): + return not self.stopping + + def process_request(self, request, client_address): + with self.workers: + self.active += 1 + try: + super().process_request(request, client_address) + except Exception: + self.worker_done() + raise + + def process_request_thread(self, request, client_address): + try: + super().process_request_thread(request, client_address) + finally: + self.worker_done() + + def worker_done(self): + with self.workers: + self.active -= 1 + self.workers.notify_all() + + def handle_error(self, request, client_address): + LOG.error("HTTP connection handler failed") + + def stop(self, signum, frame): + self.stopping = True + + def run(self): + while not self.stopping: + self.handle_request() + self.server_close() + with self.workers: + self.workers.wait_for(lambda: self.active == 0, timeout=5) + self.app.close() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + host = os.getenv("APP_HOST", "0.0.0.0") + port = int(os.getenv("APP_PORT", "8080")) + environment = os.getenv("APP_ENV", "lab") + with Server((host, port)) as server: + signal.signal(signal.SIGTERM, server.stop) + signal.signal(signal.SIGINT, server.stop) + LOG.info("started host=%s port=%s env=%s", host, server.server_port, environment) + server.run() + LOG.info("stopped") + + +if __name__ == "__main__": + main() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..b78f0ab0 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,232 @@ +import base64 +import hashlib +import hmac +import json +import logging +import secrets +from datetime import datetime +from threading import BoundedSemaphore +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from config import positive_int, secret +from profile_cache import ProfileCache +from validation import ApiError, invalid, not_found + +USER = "id, username, display_name, created_at" +MESSAGE = "id, chat_id, sender_id, text, created_at" +CHAT = """c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY position) AS member_ids""" +DB_UNAVAILABLE = (psycopg.OperationalError, PoolTimeout, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable) +LOG = logging.getLogger("messenger") + + +def json_value(value): + if isinstance(value, datetime): + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + if isinstance(value, UUID): + return str(value) + raise TypeError(type(value).__name__) + + +def unavailable(): + return ApiError(503, "unavailable", "Dependency temporarily unavailable") + + +def unauthorized(scheme): + challenge = f'{scheme} realm="messenger"' + if scheme == "Basic": + challenge += ', charset="UTF-8"' + return ApiError(401, "unauthorized", "Invalid credentials", {"WWW-Authenticate": challenge}) + + +class Store: + def __init__(self): + self.pool = ConnectionPool( + secret("DATABASE_URL"), min_size=1, max_size=8, max_waiting=24, timeout=1.5, + kwargs={"row_factory": dict_row, "connect_timeout": 2, + "options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"}, + open=True, check=ConnectionPool.check_connection, + ) + self.cache = ProfileCache(secret("CACHE_URL")) + self.cursor_key = bytes.fromhex(secret("CURSOR_KEY")) + self.ttl = positive_int("SESSION_TTL_SECONDS", 3600) + self.idempotency_ttl = positive_int("IDEMPOTENCY_TTL_SECONDS", 86400) + self.hash_slots = BoundedSemaphore(4) + self.dummy_hash = bytes(16 + 32) + + def close(self): + self.cache.close() + self.pool.close() + + def ready(self): + with self.pool.connection() as db: + db.execute("SELECT 1 FROM users LIMIT 1") + + def password_hash(self, password, salt): + if not self.hash_slots.acquire(timeout=0.5): + raise unavailable() + try: + return salt + hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=2**15, r=8, p=1, + maxmem=64 * 1024**2, dklen=32, + ) + finally: + self.hash_slots.release() + + def register(self, username, name, password): + password_hash = self.password_hash(password, secrets.token_bytes(16)) + try: + with self.pool.connection() as db: + return db.execute(f""" + INSERT INTO users (username,display_name,password_hash) VALUES (%s,%s,%s) + RETURNING {USER} + """, (username, name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Username already exists") from None + + def login(self, username, password): + with self.pool.connection() as db: + row = db.execute(f"SELECT {USER},password_hash FROM users WHERE username=%s", + (username,)).fetchone() + stored = row.pop("password_hash") if row else self.dummy_hash + matches = hmac.compare_digest(self.password_hash(password, stored[:16]), stored) + if not matches or row is None: + raise unauthorized("Basic") + token = secrets.token_urlsafe(32) + digest = self.token_hash(token) + with self.pool.connection() as db: + db.execute("DELETE FROM sessions WHERE expires_at <= now()") + session = db.execute(""" + INSERT INTO sessions (token_hash,user_id,expires_at) + VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') + RETURNING expires_at + """, (digest, row["id"], self.ttl)).fetchone() + return {"token": token, "token_type": "Bearer", **session, "user": row} + + @staticmethod + def token_hash(token): + return hashlib.sha256(token.encode("ascii")).digest() + + def actor(self, digest): + with self.pool.connection() as db: + row = db.execute(""" + SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp() + """, (digest,)).fetchone() + if row is None: + raise unauthorized("Bearer") + return row["user_id"] + + def user(self, user_id): + key = f"user:v1:{user_id}" + value = self.cache.get(key) + if value is not None: + try: + return json.loads(value) + except ValueError: + LOG.info("invalid profile cache value") + with self.pool.connection() as db: + user = db.execute(f"SELECT {USER} FROM users WHERE id=%s", (user_id,)).fetchone() + self.cache.set(key, json.dumps(user, default=json_value)) + return user + + def logout(self, digest): + with self.pool.connection() as db: + removed = db.execute(""" + DELETE FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp() + """, (digest,)).rowcount + if not removed: + raise unauthorized("Bearer") + + def create_chat(self, title, members): + try: + with self.pool.connection() as db: + chat = db.execute("INSERT INTO chats (title) VALUES (%s) RETURNING *", + (title,)).fetchone() + with db.cursor() as cursor: + cursor.executemany("INSERT INTO members (chat_id,user_id,position) VALUES (%s,%s,%s)", + [(chat["id"], member, index) for index, member in enumerate(members)]) + return {**chat, "member_ids": members} + except psycopg.errors.ForeignKeyViolation: + raise invalid("Unknown member") from None + + def chat(self, actor, chat_id): + with self.pool.connection() as db: + row = db.execute(f""" + SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id + WHERE m.user_id=%s AND c.id=%s + """, (actor, chat_id)).fetchone() + if row is None: + raise not_found() + return row + + def send(self, actor, chat_id, text, key): + fingerprint = hashlib.sha256(text.encode("utf-8")).digest() + identity = (actor, chat_id, key) + with self.pool.connection() as db: + db.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", + (f"{actor}:{chat_id}:{key}",)) + request = db.execute(""" + SELECT fingerprint,message_id FROM message_requests + WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at > clock_timestamp() + """, identity).fetchone() + if request is not None: + if not hmac.compare_digest(request["fingerprint"], fingerprint): + raise ApiError(409, "conflict", "Idempotency key already used for another body") + return db.execute(f"SELECT {MESSAGE} FROM messages WHERE id=%s", + (request["message_id"],)).fetchone() + message = db.execute(f""" + INSERT INTO messages (chat_id,sender_id,text) VALUES (%s,%s,%s) RETURNING {MESSAGE} + """, (chat_id, actor, text)).fetchone() + db.execute(""" + INSERT INTO message_requests (actor_id,chat_id,key,fingerprint,message_id,expires_at) + VALUES (%s,%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second') + ON CONFLICT (actor_id,chat_id,key) DO UPDATE + SET fingerprint=excluded.fingerprint,message_id=excluded.message_id,expires_at=excluded.expires_at + """, (*identity, fingerprint, message["id"], self.idempotency_ttl)) + return message + + def chat_page(self, actor, limit, cursor): + return self.page(f"""SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id + WHERE m.user_id=%s""", "c.", ("chats", actor), limit, cursor) + + def message_page(self, chat_id, limit, cursor): + return self.page(f"SELECT {MESSAGE} FROM messages WHERE chat_id=%s", + "", ("messages", chat_id), limit, cursor) + + def page(self, query, prefix, scope, limit, cursor): + columns = f"{prefix}created_at, {prefix}id" + params = [scope[1]] + if cursor is not None: + params.extend(self.decode_cursor(cursor, scope)) + query += f" AND ({columns}) > (%s,%s)" + with self.pool.connection() as db: + rows = db.execute(f"{query} ORDER BY {columns} LIMIT %s", (*params, limit + 1)).fetchall() + items = rows[:limit] + next_cursor = self.encode_cursor(scope, items[-1]) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} + + def encode_cursor(self, scope, row): + payload = json.dumps([scope, row["created_at"], row["id"]], + separators=(",", ":"), default=json_value).encode("ascii") + signature = hmac.digest(self.cursor_key, payload, "sha256") + return base64.urlsafe_b64encode(signature + payload).decode("ascii").rstrip("=") + + def decode_cursor(self, cursor, scope): + try: + if not 1 <= len(cursor) <= 256 or not cursor.isascii(): + raise ValueError + data = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + signature, payload = data[:32], data[32:] + if not hmac.compare_digest(signature, hmac.digest(self.cursor_key, payload, "sha256")): + raise ValueError + saved_scope, timestamp, record = json.loads(payload) + if saved_scope != [scope[0], str(scope[1])]: + raise ValueError + return datetime.fromisoformat(timestamp), UUID(record) + except ValueError: + raise invalid("Invalid cursor") from None diff --git a/src/validation.py b/src/validation.py new file mode 100644 index 00000000..66b881aa --- /dev/null +++ b/src/validation.py @@ -0,0 +1,43 @@ +import re +from uuid import UUID + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status = status + self.code = code + self.headers = headers or {} + + +def invalid(message): + return ApiError(400, "invalid_request", message) + + +def not_found(): + return ApiError(404, "not_found", "Resource not found") + + +def canonical_id(value): + if not isinstance(value, str) or not re.fullmatch( + r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value + ): + raise invalid("Invalid UUID") + return UUID(value) + + +def require_fields(body, *fields): + if not isinstance(body, dict) or body.keys() != set(fields): + raise invalid("Invalid request fields") + + +def require_text(value, maximum): + if ( + not isinstance(value, str) + or not value.strip() + or len(value) > maximum + or "\x00" in value + or re.search(r"[\ud800-\udfff]", value) + ): + raise invalid("Invalid text field") + return value diff --git a/student-tests/check_replicas.py b/student-tests/check_replicas.py new file mode 100644 index 00000000..f5941001 --- /dev/null +++ b/student-tests/check_replicas.py @@ -0,0 +1,118 @@ +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import subprocess +import sys +from threading import Barrier, Event +import time +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, check + +REPLICAS = ("api-a", "api-b") + + +def compose(*args): + return subprocess.check_output(["docker", "compose", *args], text=True, timeout=45).strip() + + +def wait_instances(expected): + client = Client(timeout=1) + deadline = time.monotonic() + 25 + seen = set() + while time.monotonic() < deadline: + try: + instance = client.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"] + seen.add(instance) + if seen == expected: + return + except Failure: + pass + time.sleep(0.1) + raise AssertionError(f"Expected {expected}, observed {seen}") + + +def parallel_replay(actor, chat): + start = Barrier(8) + key = uuid.uuid4().hex + + def write(_): + start.wait(timeout=10) + return Client().message(actor, chat, "Concurrent replay", key=key) + + with ThreadPoolExecutor(max_workers=8) as workers: + results = list(workers.map(write, range(8))) + ids = {item["json"]["id"] for item in results} + instances = {item["headers"]["X-Instance-Id"] for item in results} + check(len(ids) == 1 and instances == set(REPLICAS), "parallel replay across replicas failed") + path = f"/api/v1/chats/{chat['id']}/messages" + page = Client().call("GET", path, actor=actor, schema="MessagePage")["json"] + check(sum(item["id"] in ids for item in page["items"]) == 1, "duplicate in history") + print("PASS: 8 concurrent replays through both replicas, 1 message", flush=True) + + +def exercise(actor, chat, target, recreate): + stop = Event() + results = [] + client = Client(timeout=2) + started = time.monotonic() + + def traffic(): + key = uuid.uuid4().hex + while not stop.is_set(): + try: + response = client.message(actor, chat, "Write " + key, key=key) + results.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"])) + key = uuid.uuid4().hex + except Failure: + results.append((time.monotonic(), None, None)) + stop.wait(0.05) + + old_id = compose("ps", "-q", target) + with ThreadPoolExecutor(max_workers=1) as workers: + worker = workers.submit(traffic) + try: + time.sleep(0.5) + changed = time.monotonic() + if recreate: + compose("up", "-d", "--no-deps", "--force-recreate", target) + check(old_id != compose("ps", "-q", target), "container was not replaced") + wait_instances(set(REPLICAS)) + else: + compose("stop", "--timeout", "5", target) + check(not compose("ps", "-q", "--status", "running", target), "replica is still running") + time.sleep(2.5) + finally: + stop.set() + worker.result(timeout=5) + if not recreate: + compose("start", target) + wait_instances(set(REPLICAS)) + + survivor = next(name for name in REPLICAS if name != target) + successful = [row for row in results if row[1] is not None] + first = next(row[0] for row in successful if row[0] >= changed and row[1] == survivor) + check(first - changed <= 10, "surviving replica took too long") + path = f"/api/v1/chats/{chat['id']}/messages?limit=100" + observed = set() + while path is not None: + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + observed.update(item["id"] for item in page["items"]) + cursor = page["next_cursor"] + path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={cursor}" if cursor else None + check({row[2] for row in successful} <= observed, "confirmed writes lost") + action = "recreate" if recreate else "stop" + elapsed = results[-1][0] - started + print(f"PASS: {action} {target}, {len(successful) / elapsed:.1f} write/s, " + f"errors={len(results) - len(successful)}, survivor={first - changed:.3f}s", flush=True) + + +if __name__ == "__main__": + compose("start", *REPLICAS) + wait_instances(set(REPLICAS)) + owner, _, _, chat = Client().bootstrap() + parallel_replay(owner, chat) + for name in REPLICAS: + exercise(owner, chat, name, recreate=False) + for name in REPLICAS: + exercise(owner, chat, name, recreate=True) -- 2.54.0