From 7329d94740039d52a67f7cdd9032faba3174c2c5 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 6 Nov 2026 20:40:12 +0300 Subject: [PATCH 1/3] finished --- .env.example | 10 +- .github/mtusi2026lab2.lnk | Bin 0 -> 1250 bytes .github/workflows/contracts.yml | 26 ++- Dockerfile | 24 ++ Makefile | 14 +- compose.yaml | 78 +++++++ migrations/001_init.sql | 51 +++++ requirements.txt | 8 + scripts/restart.py | 21 ++ scripts/restart.sh | 9 + scripts/run_expiry.py | 51 +++++ src/app/__init__.py | 1 + src/app/cache.py | 133 +++++++++++ src/app/config.py | 33 +++ src/app/context.py | 8 + src/app/cursors.py | 35 +++ src/app/db.py | 98 +++++++++ src/app/errors.py | 25 +++ src/app/main.py | 76 +++++++ src/app/migrate.py | 38 ++++ src/app/passwords.py | 23 ++ src/app/rediscli.py | 44 ++++ src/app/routes.py | 378 ++++++++++++++++++++++++++++++++ src/app/serialize.py | 31 +++ src/app/sessions.py | 56 +++++ src/app/timeutil.py | 12 + src/app/tools/__init__.py | 0 src/app/tools/explain.py | 85 +++++++ student-tests/expiry.py | 22 ++ student-tests/infra.py | 92 ++++++++ student-tests/run.py | 184 ++++++++++++++++ 31 files changed, 1662 insertions(+), 4 deletions(-) create mode 100644 .github/mtusi2026lab2.lnk create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 migrations/001_init.sql create mode 100644 requirements.txt create mode 100644 scripts/restart.py create mode 100644 scripts/restart.sh create mode 100644 scripts/run_expiry.py create mode 100644 src/app/__init__.py create mode 100644 src/app/cache.py create mode 100644 src/app/config.py create mode 100644 src/app/context.py create mode 100644 src/app/cursors.py create mode 100644 src/app/db.py create mode 100644 src/app/errors.py create mode 100644 src/app/main.py create mode 100644 src/app/migrate.py create mode 100644 src/app/passwords.py create mode 100644 src/app/rediscli.py create mode 100644 src/app/routes.py create mode 100644 src/app/serialize.py create mode 100644 src/app/sessions.py create mode 100644 src/app/timeutil.py create mode 100644 src/app/tools/__init__.py create mode 100644 src/app/tools/explain.py create mode 100644 student-tests/expiry.py create mode 100644 student-tests/infra.py create mode 100644 student-tests/run.py diff --git a/.env.example b/.env.example index 714c8076..e065c373 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,16 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 +CACHE_TTL_SECONDS=60 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url + +# Compose interpolates these. Synthetic local values only; replace after copy. +POSTGRES_PASSWORD=lab2-dev-only-change-me +CURSOR_SIGNING_KEY=lab2-dev-cursor-key-change-me-32bytes +# Application also accepts DATABASE_URL / REDIS_URL (compose injects them). +DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 diff --git a/.github/mtusi2026lab2.lnk b/.github/mtusi2026lab2.lnk new file mode 100644 index 0000000000000000000000000000000000000000..5cfc18e4e1271ef0fda02d8a455297549470a6d8 GIT binary patch literal 1250 zcmai!Ur1AN6vxl(KB$$=Tq0s^Scy{FocL!`OQ-0pWr7Koci`OIpPX%Yr}&bik%XiW zL=XkhA4`IQgd~JkK12jXNDoDpf!IUCa`;p)Qr~murfJk2=iK}Io%8$r&i$VIo0*6d zsswnVDcMuq1xh3(j{|LGn@`Uwp45$0@%^)ORvybi#1+x}xWA$>klJK;{p`{Hk7nb; zyg>Ns`4`nW$jhO~(^TYTO-8iar#~siLc3i9)>%bXD5WSFWEC}0q0DKr%%4zFx4+ls zrV2_T0V^9UU)vlOAy|NntqYvCQ_fv=fr~!sCO`F(jog@4(a9B_!;%(MlFRLMdAwFT zkulDUfSD#UOTI9vXcZ?vO&r#f3p6LZhrDDZET;@=jFT5Z_ENV z{EJmhUq=l@Zh>Z{4#G0jX^+#EK|6Z9rMW>jfRH&(Tn2GjChD&-taTkR`#_>1`tA~{ z)sYrfvSyDA~3N{e3oPG0n99px(rJ^WdA{w|-UI z+@87ChV=JKsh74qugnZjF8wdQ54t9Se-rx@Ha9hY*M??lXsCYd=XBbrG~JF+y06e6P@5xtcx(9@h35jbOM4$j>NAyhmJ-^BuP@wZ^^-Y+3rvCit9?5V-v*;9cDIN0 zd=mMr2s~y2H%*OF4+#c%E>lCbc+#U(tx$pDiI~DW$&~mx#yBS8-BY&Q%FAz4C*IN2 zu@4#3EaEO+5EC&fTnkCmNj>m?N>z(liMntEz2rk|^)PM}NQ^fmmiPA$mv@ggkVPC% s@2-6?;xmnzstQIc`-k~2z{LqB*m#Z*di64FJQs47pJ}{T$Mdo4H}A>^d;kCd literal 0 HcmV?d00001 diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289f..7e020ea6 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,5 +12,27 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + application: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: cp .env.example .env + - run: docker compose --env-file .env up -d --build + - name: Wait until API is ready + run: | + for i in $(seq 1 60); do + if curl -fsS http://127.0.0.1:8080/health/ready; then exit 0; fi + sleep 2 + done + docker compose --env-file .env logs + exit 1 + - run: make test PYTHON=python + - run: chmod +x scripts/restart.sh + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python + - run: python student-tests/run.py + - run: python student-tests/infra.py + - run: python scripts/run_expiry.py + - run: docker compose --env-file .env exec -T api python -m app.tools.explain diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1b537886 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12.8-slim-bookworm + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser \ + && apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY src/app /app/app +COPY migrations /app/migrations + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/app + +USER appuser +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \ + CMD curl -fsS http://127.0.0.1:8080/health/ready || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Makefile b/Makefile index 9ecb223e..bc3e2382 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence test-student test-infra test-expiry explain help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,3 +24,15 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-student: + $(PYTHON) student-tests/run.py + +test-infra: + $(PYTHON) student-tests/infra.py + +test-expiry: + $(PYTHON) scripts/run_expiry.py + +explain: + docker compose --env-file .env exec -T api python -m app.tools.explain diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..0addf266 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,78 @@ +services: + api: + build: + context: . + dockerfile: Dockerfile + image: mtusi-lab2-api:1.2.0 + ports: + - "8080:8080" + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60} + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + REDIS_URL: redis://redis:6379/0 + CURSOR_SIGNING_KEY: ${CURSOR_SIGNING_KEY} + DB_POOL_MIN: "1" + DB_POOL_MAX: "10" + DB_COMMAND_TIMEOUT: "5" + DB_CONNECT_TIMEOUT: "5" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + mem_limit: 512m + cpus: "1.0" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 20s + + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: messenger + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 512m + cpus: "1.0" + + redis: + image: redis:7.4.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --save + - "60" + - "1" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 128m + cpus: "0.50" + +volumes: + pg_data: + redis_data: diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..12b88622 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,51 @@ +-- Lab2 schema. PostgreSQL is the source of truth for users, chats, members, messages. +-- Transaction boundary: create-chat inserts the chat row and all members in one transaction. +-- Sessions live in Redis (AOF); this database does not store session tokens. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE UNIQUE INDEX users_username_key ON users (username); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id); + +-- List chats of a member ordered by (created_at, id). +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset pagination: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n +-- This composite index avoids a sequential scan of the whole history per page. +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d5ab2583 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +fastapi==0.115.6 +uvicorn==0.32.1 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==23.1.0 +starlette==0.41.3 +pydantic==2.10.3 +python-multipart==0.0.20 diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..fb4c78d8 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Restart API, PostgreSQL and Redis without removing named volumes.""" +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(["restart", "postgres", "redis", "api"]) + proc = subprocess.run(cmd, cwd=ROOT) + return proc.returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..81d27153 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +if [ -f .env ]; then + docker compose --env-file .env restart postgres redis api +else + docker compose restart postgres redis api +fi diff --git a/scripts/run_expiry.py b/scripts/run_expiry.py new file mode 100644 index 00000000..a5f51c4c --- /dev/null +++ b/scripts/run_expiry.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL.""" +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(args, extra_env=None): + env = os.environ.copy() + if extra_env: + env.update(extra_env) + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, env=env) + if proc.returncode != 0: + raise SystemExit(proc.returncode) + + +def wait_ready(): + import urllib.request + + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) + return + except Exception: + time.sleep(0.5) + raise SystemExit("API did not become ready") + + +def main(): + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"}) + wait_ready() + env = os.environ.copy() + env["SESSION_TTL_SECONDS"] = "2" + env["RUN_EXPIRY"] = "1" + proc = subprocess.run([sys.executable, str(ROOT / "student-tests" / "expiry.py")], cwd=ROOT, env=env) + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"}) + wait_ready() + raise SystemExit(proc.returncode) + + +if __name__ == "__main__": + main() diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 00000000..045965c1 --- /dev/null +++ b/src/app/__init__.py @@ -0,0 +1 @@ +"""MTUSI messenger API, laboratory 2.""" diff --git a/src/app/cache.py b/src/app/cache.py new file mode 100644 index 00000000..2dc8bd5f --- /dev/null +++ b/src/app/cache.py @@ -0,0 +1,133 @@ +import asyncio +import json +from collections.abc import Awaitable, Callable +from typing import TypeVar + +from . import rediscli +from .config import settings +from .errors import AppError + +T = TypeVar("T") + +CFG_SINGLEFLIGHT = "cfg:singleflight" +CFG_INVALIDATE = "cfg:invalidate" +SQL_HISTORY = "metrics:sql:messages_select" + + +class SingleFlight: + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._inflight: dict[str, asyncio.Task] = {} + + async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T: + async with self._lock: + task = self._inflight.get(key) + if task is None: + task = asyncio.create_task(factory()) + self._inflight[key] = task + owner = True + else: + owner = False + try: + return await asyncio.shield(task) + finally: + if owner: + async with self._lock: + self._inflight.pop(key, None) + + +_sf = SingleFlight() + + +async def _flag(name: str, default: bool = True) -> bool: + if not settings.lab_mode: + return default + try: + raw = await rediscli.require().get(name) + except Exception: + return default + if raw is None: + return default + return raw not in {"0", "false", "off"} + + +async def singleflight_enabled() -> bool: + return await _flag(CFG_SINGLEFLIGHT, True) + + +async def invalidate_enabled() -> bool: + return await _flag(CFG_INVALIDATE, True) + + +async def incr_history_sql() -> None: + try: + await rediscli.require().incr(SQL_HISTORY) + except Exception: + pass + + +async def history_sql_count() -> int: + try: + raw = await rediscli.require().get(SQL_HISTORY) + return int(raw or 0) + except Exception: + return 0 + + +async def reset_history_sql_count() -> None: + try: + await rediscli.require().set(SQL_HISTORY, 0) + except Exception: + pass + + +async def set_flags(*, singleflight: bool | None = None, invalidate: bool | None = None) -> dict: + r = rediscli.require() + if singleflight is not None: + await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0") + if invalidate is not None: + await r.set(CFG_INVALIDATE, "1" if invalidate else "0") + return { + "singleflight": await singleflight_enabled(), + "invalidate": await invalidate_enabled(), + "history_sql": await history_sql_count(), + } + + +def _page_key(chat_id: str, version: str, cursor: str, limit: int) -> str: + return f"hist:{chat_id}:{version}:{cursor}:{limit}" + + +async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict: + r = rediscli.require() + try: + version = await r.get("histver:" + chat_id) or "0" + key = _page_key(chat_id, version, cursor, limit) + cached = await r.get(key) + if cached: + return json.loads(cached) + + async def load_and_store() -> dict: + page = await loader() + try: + await r.set(key, json.dumps(page, ensure_ascii=False), ex=settings.cache_ttl_seconds) + except Exception: + pass + return page + + if await singleflight_enabled(): + return await _sf.do(key, load_and_store) + return await load_and_store() + except AppError: + raise + except Exception: + return await loader() + + +async def invalidate_history(chat_id: str) -> None: + if not await invalidate_enabled(): + return + try: + await rediscli.require().incr("histver:" + chat_id) + except Exception: + pass diff --git a/src/app/config.py b/src/app/config.py new file mode 100644 index 00000000..f7f4ed52 --- /dev/null +++ b/src/app/config.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path + + +def _file_or_env(name: str, default: str = "") -> str: + path = os.environ.get(name + "_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get(name, default) + + +class Settings: + def __init__(self) -> None: + self.app_host = os.environ.get("APP_HOST", "0.0.0.0") + self.app_port = int(os.environ.get("APP_PORT", "8080")) + self.app_env = os.environ.get("APP_ENV", "lab") + self.session_ttl_seconds = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + self.cache_ttl_seconds = int(os.environ.get("CACHE_TTL_SECONDS", "60")) + self.database_url = _file_or_env("DATABASE_URL") + self.redis_url = _file_or_env("REDIS_URL", "redis://redis:6379/0") + self.cursor_signing_key = _file_or_env("CURSOR_SIGNING_KEY", "lab2-dev-cursor-key-change-me") + self.db_pool_min = int(os.environ.get("DB_POOL_MIN", "1")) + self.db_pool_max = int(os.environ.get("DB_POOL_MAX", "10")) + self.db_command_timeout = float(os.environ.get("DB_COMMAND_TIMEOUT", "5")) + self.db_connect_timeout = float(os.environ.get("DB_CONNECT_TIMEOUT", "5")) + self.redis_socket_timeout = float(os.environ.get("REDIS_SOCKET_TIMEOUT", "2")) + + @property + def lab_mode(self) -> bool: + return self.app_env == "lab" + + +settings = Settings() diff --git a/src/app/context.py b/src/app/context.py new file mode 100644 index 00000000..632762bd --- /dev/null +++ b/src/app/context.py @@ -0,0 +1,8 @@ +from contextvars import ContextVar + +request_id_var: ContextVar[str] = ContextVar("request_id", default="") +sql_count_var: ContextVar[int] = ContextVar("sql_count", default=0) + + +def bump_sql(n: int = 1) -> None: + sql_count_var.set(sql_count_var.get() + n) diff --git a/src/app/cursors.py b/src/app/cursors.py new file mode 100644 index 00000000..c1715ce1 --- /dev/null +++ b/src/app/cursors.py @@ -0,0 +1,35 @@ +import base64 +import hashlib +import hmac +import json + +from .config import settings +from .errors import AppError + + +def _sign(payload: str) -> str: + key = settings.cursor_signing_key.encode("utf-8") + return hmac.new(key, payload.encode("utf-8"), hashlib.sha256).hexdigest()[:24] + + +def encode_cursor(data: dict) -> str: + payload = json.dumps(data, separators=(",", ":"), ensure_ascii=False) + raw = payload + "." + _sign(payload) + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_cursor(token: str, kind: str, chat_id: str | None = None) -> dict: + try: + padded = token + "=" * (-len(token) % 4) + raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8") + payload, _, sig = raw.partition(".") + if not sig or not hmac.compare_digest(sig, _sign(payload)): + raise ValueError("bad signature") + data = json.loads(payload) + if data.get("k") != kind: + raise ValueError("wrong kind") + if chat_id is not None and data.get("c") != chat_id: + raise ValueError("wrong chat") + return data + except Exception: + raise AppError(400, "invalid_request", "Некорректный cursor") from None diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 00000000..3f4e2d39 --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,98 @@ +import asyncpg + +from .config import settings +from .context import bump_sql +from .errors import AppError + +pool: asyncpg.Pool | None = None + + +async def connect() -> asyncpg.Pool: + global pool + if not settings.database_url: + raise RuntimeError("DATABASE_URL is not set") + pool = await asyncpg.create_pool( + dsn=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + command_timeout=settings.db_command_timeout, + timeout=settings.db_connect_timeout, + ) + return pool + + +async def close() -> None: + global pool + if pool is not None: + await pool.close() + pool = None + + +def require_pool() -> asyncpg.Pool: + if pool is None: + raise AppError(503, "unavailable", "Сервис временно недоступен") + return pool + + +async def ping() -> None: + async with require_pool().acquire() as conn: + bump_sql() + await conn.execute("SELECT 1") + + +_INTEGRITY = ( + asyncpg.UniqueViolationError, + asyncpg.ForeignKeyViolationError, + asyncpg.CheckViolationError, +) + + +async def fetch(sql: str, *args, count: bool = True): + try: + async with require_pool().acquire() as conn: + if count: + bump_sql() + return await conn.fetch(sql, *args) + except AppError: + raise + except _INTEGRITY: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +async def fetchrow(sql: str, *args, count: bool = True): + try: + async with require_pool().acquire() as conn: + if count: + bump_sql() + return await conn.fetchrow(sql, *args) + except AppError: + raise + except _INTEGRITY: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +async def execute(sql: str, *args, count: bool = True): + try: + async with require_pool().acquire() as conn: + if count: + bump_sql() + return await conn.execute(sql, *args) + except AppError: + raise + except _INTEGRITY: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +async def transaction_connection(): + try: + return require_pool().acquire() + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None diff --git a/src/app/errors.py b/src/app/errors.py new file mode 100644 index 00000000..f41964ce --- /dev/null +++ b/src/app/errors.py @@ -0,0 +1,25 @@ +from fastapi.responses import JSONResponse + + +class AppError(Exception): + def __init__(self, status: int, code: str, message: str, www_authenticate: str | None = None) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.www_authenticate = www_authenticate + + +def error_response(status: int, code: str, message: str, request_id: str, www_authenticate: str | None = None) -> JSONResponse: + headers = {"X-Request-Id": request_id} + if www_authenticate: + headers["WWW-Authenticate"] = www_authenticate + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers, + ) + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 00000000..ad54f3f5 --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,76 @@ +from contextlib import asynccontextmanager +import uuid + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import db, rediscli +from .config import settings +from .context import request_id_var, sql_count_var +from .errors import AppError, error_response +from .migrate import apply_migrations +from .routes import api, health + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + await db.connect() + await apply_migrations(db.require_pool()) + await rediscli.connect() + yield + await rediscli.close() + await db.close() + + +app = FastAPI(title="MTUSI Messenger", version="1.2.0", lifespan=lifespan, redirect_slashes=False) +app.include_router(health) +app.include_router(api, prefix="/api/v1") + + +@app.middleware("http") +async def request_context(request: Request, call_next): + rid = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = rid + token_rid = request_id_var.set(rid) + token_sql = sql_count_var.set(0) + sql_count = 0 + try: + response = await call_next(request) + sql_count = sql_count_var.get() + finally: + request_id_var.reset(token_rid) + sql_count_var.reset(token_sql) + response.headers["X-Request-Id"] = rid + if settings.lab_mode: + response.headers["X-Sql-Count"] = str(sql_count) + return response + + +@app.exception_handler(AppError) +async def handle_app_error(request: Request, exc: AppError): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + return error_response(exc.status, exc.code, exc.message, rid, exc.www_authenticate) + + +@app.exception_handler(RequestValidationError) +async def handle_validation(request: Request, _exc: RequestValidationError): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + return error_response(400, "invalid_request", "Некорректный запрос", rid) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http(request: Request, exc: StarletteHTTPException): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + if exc.status_code == 404: + return error_response(404, "not_found", "Объект не найден", rid) + if exc.status_code == 405: + return error_response(400, "invalid_request", "Некорректный запрос", rid) + return error_response(exc.status_code, "internal_error", "Внутренняя ошибка", rid) + + +@app.exception_handler(Exception) +async def handle_unexpected(request: Request, _exc: Exception): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + return error_response(500, "internal_error", "Внутренняя ошибка", rid) diff --git a/src/app/migrate.py b/src/app/migrate.py new file mode 100644 index 00000000..9215e7c9 --- /dev/null +++ b/src/app/migrate.py @@ -0,0 +1,38 @@ +from pathlib import Path + +import asyncpg + + +def _migrations_dir() -> Path: + here = Path(__file__).resolve().parent + for candidate in (here.parent.parent / "migrations", Path("/app/migrations")): + if candidate.is_dir(): + return candidate + raise RuntimeError("migrations directory not found") + + +MIGRATIONS = _migrations_dir() + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + async with pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")} + files = sorted(p for p in MIGRATIONS.glob("*.sql") if p.is_file()) + for path in files: + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + "INSERT INTO schema_migrations(filename) VALUES($1)", + path.name, + ) diff --git a/src/app/passwords.py b/src/app/passwords.py new file mode 100644 index 00000000..8431b234 --- /dev/null +++ b/src/app/passwords.py @@ -0,0 +1,23 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + +hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1) +_dummy_hash = None + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def dummy_hash() -> str: + global _dummy_hash + if _dummy_hash is None: + _dummy_hash = hash_password("timing-dummy-password") + return _dummy_hash + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError, ValueError, TypeError): + return False diff --git a/src/app/rediscli.py b/src/app/rediscli.py new file mode 100644 index 00000000..e38c6b5b --- /dev/null +++ b/src/app/rediscli.py @@ -0,0 +1,44 @@ +import redis.asyncio as redis + +from .config import settings +from .errors import AppError + +client: redis.Redis | None = None + + +async def connect() -> redis.Redis: + global client + client = redis.from_url( + settings.redis_url, + socket_connect_timeout=settings.redis_socket_timeout, + socket_timeout=settings.redis_socket_timeout, + decode_responses=True, + ) + await client.ping() + return client + + +async def close() -> None: + global client + if client is not None: + await client.close() + client = None + + +def require() -> redis.Redis: + if client is None: + raise AppError(503, "unavailable", "Сервис временно недоступен") + return client + + +async def ping() -> None: + try: + await require().ping() + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +def wrap_redis_errors(exc: Exception) -> AppError: + return AppError(503, "unavailable", "Сервис временно недоступен") diff --git a/src/app/routes.py b/src/app/routes.py new file mode 100644 index 00000000..e80b9de8 --- /dev/null +++ b/src/app/routes.py @@ -0,0 +1,378 @@ +import base64 +import binascii +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import asyncpg +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse, Response +from pydantic import BaseModel, ConfigDict, Field + +from . import cache, db, rediscli, sessions +from .cursors import decode_cursor, encode_cursor +from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError +from .config import settings +from .passwords import dummy_hash, hash_password, verify_password +from .serialize import chat_json, message_json, user_json +from .timeutil import fmt_dt + +api = APIRouter() +health = APIRouter() + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(min_length=1, max_length=4000) + + +def _blank(value: str) -> bool: + return value.strip() == "" + + +def _limit(raw: str | None) -> int: + if raw is None or raw == "": + return 50 + try: + value = int(raw) + except ValueError: + raise AppError(400, "invalid_request", "Некорректный limit") from None + if value < 1 or value > 100: + raise AppError(400, "invalid_request", "Некорректный limit") + return value + + +async def current_user(request: Request) -> asyncpg.Record: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + user_id = await sessions.resolve(rest.strip()) + row = await db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id) + if row is None: + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + return row + + +async def member_ids_of(chat_id: UUID) -> list[UUID]: + rows = await db.fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id) + return [r["user_id"] for r in rows] + + +async def load_chat_for_member(chat_id: UUID, user_id: UUID) -> asyncpg.Record: + row = await db.fetchrow( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 + WHERE c.id = $1 + """, + chat_id, + user_id, + ) + if row is None: + raise AppError(404, "not_found", "Объект не найден") + return row + + +@health.get("/health/live") +async def live() -> JSONResponse: + return JSONResponse({"status": "ok"}) + + +@health.get("/health/ready") +async def ready() -> JSONResponse: + await db.ping() + await rediscli.ping() + return JSONResponse({"status": "ok"}) + + +@api.post("/users") +async def register(body: CreateUser) -> JSONResponse: + if _blank(body.display_name) or _blank(body.password) or _blank(body.username): + raise AppError(400, "invalid_request", "Некорректные поля") + user_id = uuid4() + created = datetime.now(timezone.utc) + try: + await db.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + user_id, + body.username, + body.display_name, + hash_password(body.password), + created, + ) + except asyncpg.UniqueViolationError: + raise AppError(409, "conflict", "Имя пользователя занято") from None + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + return JSONResponse( + user_json({"id": user_id, "username": body.username, "display_name": body.display_name, "created_at": created}), + status_code=201, + ) + + +@api.get("/users/me") +async def me(request: Request) -> JSONResponse: + row = await current_user(request) + return JSONResponse(user_json(row)) + + +@api.post("/auth/sessions") +async def login(request: Request) -> JSONResponse: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "basic" or not rest.strip(): + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) + try: + decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeError, ValueError): + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) from None + username, sep, password = decoded.partition(":") + if not sep: + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) + row = await db.fetchrow( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1", + username, + ) + password_hash = row["password_hash"] if row else dummy_hash() + if row is None or not verify_password(password_hash, password): + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) + token, expires_at = await sessions.create(row["id"]) + return JSONResponse( + { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": user_json(row), + }, + status_code=201, + ) + + +@api.delete("/auth/sessions/current") +async def logout(request: Request) -> Response: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + await sessions.revoke(rest.strip()) + return Response(status_code=204) + + +@api.post("/chats") +async def create_chat(request: Request, body: CreateChat) -> JSONResponse: + if _blank(body.title): + raise AppError(400, "invalid_request", "Некорректные поля") + if len(body.member_ids) != len(set(body.member_ids)): + raise AppError(400, "invalid_request", "Некорректный список участников") + user = await current_user(request) + if user["id"] not in body.member_ids: + raise AppError(400, "invalid_request", "Некорректный список участников") + chat_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require_pool().acquire() as conn: + async with conn.transaction(): + existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", body.member_ids) + if len(existing) != len(body.member_ids): + raise AppError(400, "invalid_request", "Некорректный список участников") + await conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", + chat_id, + body.title, + created, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, mid) for mid in body.member_ids], + ) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + return JSONResponse(chat_json({"id": chat_id, "title": body.title, "created_at": created}, body.member_ids), status_code=201) + + +@api.get("/chats") +async def list_chats(request: Request) -> JSONResponse: + user = await current_user(request) + limit = _limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + args: list = [user["id"]] + clause = "" + if cursor: + data = decode_cursor(cursor, "chat") + clause = "AND (c.created_at, c.id) > ($2, $3)" + args.extend([datetime.fromisoformat(data["t"].replace("Z", "+00:00")), UUID(data["i"])]) + rows = await db.fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1 + WHERE 1=1 {clause} + ORDER BY c.created_at, c.id + LIMIT {limit + 1} + """, + *args, + ) + page = rows[:limit] + items = [] + for row in page: + items.append(chat_json(row, await member_ids_of(row["id"]))) + next_cursor = None + if len(rows) > limit: + last = page[-1] + next_cursor = encode_cursor({"k": "chat", "t": fmt_dt(last["created_at"]), "i": str(last["id"])}) + return JSONResponse({"items": items, "next_cursor": next_cursor}) + + +@api.get("/chats/{chat_id}") +async def get_chat(request: Request, chat_id: UUID) -> JSONResponse: + user = await current_user(request) + row = await load_chat_for_member(chat_id, user["id"]) + return JSONResponse(chat_json(row, await member_ids_of(chat_id))) + + +async def _load_messages(chat_id: UUID, after_ts: datetime | None, after_id: UUID | None, limit: int) -> dict: + await cache.incr_history_sql() + if after_ts is None: + rows = await db.fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id + LIMIT $2 + """, + chat_id, + limit + 1, + ) + else: + rows = await db.fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT $4 + """, + chat_id, + after_ts, + after_id, + limit + 1, + ) + page = rows[:limit] + items = [message_json(r) for r in page] + next_cursor = None + if len(rows) > limit: + last = page[-1] + next_cursor = encode_cursor( + {"k": "msg", "c": str(chat_id), "t": fmt_dt(last["created_at"]), "i": str(last["id"])} + ) + return {"items": items, "next_cursor": next_cursor} + + +@api.post("/chats/{chat_id}/messages") +async def create_message(request: Request, chat_id: UUID, body: CreateMessage) -> JSONResponse: + if _blank(body.text): + raise AppError(400, "invalid_request", "Некорректные поля") + user = await current_user(request) + await load_chat_for_member(chat_id, user["id"]) + message_id = uuid4() + created = datetime.now(timezone.utc) + try: + await db.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + message_id, + chat_id, + user["id"], + body.text, + created, + ) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + await cache.invalidate_history(str(chat_id)) + return JSONResponse( + message_json( + { + "id": message_id, + "chat_id": chat_id, + "sender_id": user["id"], + "text": body.text, + "created_at": created, + } + ), + status_code=201, + ) + + +@api.get("/chats/{chat_id}/messages") +async def list_messages(request: Request, chat_id: UUID) -> JSONResponse: + user = await current_user(request) + await load_chat_for_member(chat_id, user["id"]) + limit = _limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") or "" + after_ts = after_id = None + if cursor: + data = decode_cursor(cursor, "msg", str(chat_id)) + after_ts = datetime.fromisoformat(data["t"].replace("Z", "+00:00")) + after_id = UUID(data["i"]) + + async def loader(): + return await _load_messages(chat_id, after_ts, after_id, limit) + + page = await cache.get_history_page(str(chat_id), cursor, limit, loader) + return JSONResponse(page) + + +class LabFlags(BaseModel): + model_config = ConfigDict(extra="forbid") + singleflight: bool | None = None + invalidate: bool | None = None + reset_sql: bool | None = None + + +@api.get("/_lab/metrics") +async def lab_metrics() -> JSONResponse: + if not settings.lab_mode: + raise AppError(404, "not_found", "Объект не найден") + return JSONResponse( + { + "singleflight": await cache.singleflight_enabled(), + "invalidate": await cache.invalidate_enabled(), + "history_sql": await cache.history_sql_count(), + } + ) + + +@api.post("/_lab/flags") +async def lab_flags(body: LabFlags) -> JSONResponse: + if not settings.lab_mode: + raise AppError(404, "not_found", "Объект не найден") + if body.reset_sql: + await cache.reset_history_sql_count() + flags = await cache.set_flags(singleflight=body.singleflight, invalidate=body.invalidate) + return JSONResponse(flags) diff --git a/src/app/serialize.py b/src/app/serialize.py new file mode 100644 index 00000000..37f03710 --- /dev/null +++ b/src/app/serialize.py @@ -0,0 +1,31 @@ +from uuid import UUID + +from .timeutil import fmt_dt + + +def user_json(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": fmt_dt(row["created_at"]), + } + + +def chat_json(row, member_ids: list[UUID]) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(x) for x in member_ids], + "created_at": fmt_dt(row["created_at"]), + } + + +def message_json(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": fmt_dt(row["created_at"]), + } diff --git a/src/app/sessions.py b/src/app/sessions.py new file mode 100644 index 00000000..bb19b760 --- /dev/null +++ b/src/app/sessions.py @@ -0,0 +1,56 @@ +import hashlib +import json +import secrets +from datetime import timedelta +from uuid import UUID + +from . import rediscli +from .config import settings +from .errors import AppError, BEARER_CHALLENGE +from .timeutil import fmt_dt, now_utc + + +def new_token() -> str: + return secrets.token_urlsafe(32) + + +def _key(token: str) -> str: + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "sess:" + digest + + +async def create(user_id: UUID) -> tuple[str, str]: + token = new_token() + expires = now_utc() + timedelta(seconds=settings.session_ttl_seconds) + payload = json.dumps({"user_id": str(user_id), "expires_at": fmt_dt(expires)}) + try: + await rediscli.require().set(_key(token), payload, ex=settings.session_ttl_seconds) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + return token, fmt_dt(expires) + + +async def resolve(token: str) -> UUID: + try: + raw = await rediscli.require().get(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + if not raw: + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + data = json.loads(raw) + return UUID(data["user_id"]) + + +async def revoke(token: str) -> None: + try: + deleted = await rediscli.require().delete(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + if not deleted: + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) diff --git a/src/app/timeutil.py b/src/app/timeutil.py new file mode 100644 index 00000000..ff98eb6d --- /dev/null +++ b/src/app/timeutil.py @@ -0,0 +1,12 @@ +from datetime import datetime, timezone + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def fmt_dt(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + value = value.astimezone(timezone.utc) + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") diff --git a/src/app/tools/__init__.py b/src/app/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/tools/explain.py b/src/app/tools/explain.py new file mode 100644 index 00000000..5178d4d1 --- /dev/null +++ b/src/app/tools/explain.py @@ -0,0 +1,85 @@ +"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for history pagination.""" +import asyncio +import os +from uuid import uuid4 + +import asyncpg + +from app.passwords import hash_password + + +async def main() -> None: + url = os.environ["DATABASE_URL"] + conn = await asyncpg.connect(dsn=url, timeout=10, command_timeout=120) + try: + alice = uuid4() + bob = uuid4() + chat = uuid4() + await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)", + alice, + "explain_alice_" + alice.hex[:8], + "Alice", + hash_password("Example-only-not-a-real-secret-123"), + bob, + "explain_bob_" + bob.hex[:8], + "Bob", + ) + await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat") + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)", + chat, + alice, + bob, + ) + n = int(os.environ.get("EXPLAIN_MESSAGES", "10000")) + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval + FROM generate_series(1, $3) AS g + """, + chat, + alice, + n, + ) + count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat) + print("seeded_messages", count, "chat_id", chat) + + query = """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT 50 + """ + first = await conn.fetchrow( + "SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1", + chat, + ) + print("\n=== WITH index messages_chat_created_id_idx ===") + plan = await conn.fetch( + "EXPLAIN (ANALYZE, BUFFERS) " + query, + chat, + first["created_at"], + first["id"], + ) + print("\n".join(row[0] for row in plan)) + + await conn.execute("SET enable_indexscan = off") + await conn.execute("SET enable_bitmapscan = off") + await conn.execute("SET enable_indexonlyscan = off") + print("\n=== WITHOUT index scans (seq scan baseline) ===") + plan2 = await conn.fetch( + "EXPLAIN (ANALYZE, BUFFERS) " + query, + chat, + first["created_at"], + first["id"], + ) + print("\n".join(row[0] for row in plan2)) + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/student-tests/expiry.py b/student-tests/expiry.py new file mode 100644 index 00000000..b242225c --- /dev/null +++ b/student-tests/expiry.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +os.environ["RUN_EXPIRY"] = "1" +from run import test_expiry # noqa: E402 + +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, run # noqa: E402 + + +def main(): + c = Client(os.environ.get("BASE_URL", "http://localhost:8080")) + test_expiry(c) + print("PASS: session expiry") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/infra.py b/student-tests/infra.py new file mode 100644 index 00000000..6a677d45 --- /dev/null +++ b/student-tests/infra.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Dependency failure and Redis-only restart checks (Docker Compose).""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run # noqa: E402 + + +def compose(*args): + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + if proc.returncode != 0: + raise Failure("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:])) + return proc + + +def wait_ready(c: Client, seconds=90): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = exc + time.sleep(0.5) + raise Failure("ready not restored: %s" % last) + + +def wait_status(c: Client, path, expected, seconds=30, actor=None): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + c.call("GET", path, expected, actor=actor, schema="Health" if expected == 200 and path.startswith("/health") else None) + return + except Failure as exc: + last = exc + time.sleep(0.4) + raise Failure("did not observe %s for %s: %s" % (expected, path, last)) + + +def main(): + base = os.environ.get("BASE_URL", "http://localhost:8080") + c = Client(base, timeout=3) + alice = c.user("infra") + compose("ps") + + compose("stop", "redis") + wait_status(c, "/health/live", 200) + wait_status(c, "/health/ready", 503) + wait_status(c, "/api/v1/users/me", 503, actor=alice) + compose("start", "redis") + wait_ready(c) + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + compose("stop", "postgres") + wait_status(c, "/health/live", 200) + wait_status(c, "/health/ready", 503) + compose("start", "postgres") + wait_ready(c) + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + alice2, bob, _eve, chat = c.bootstrap() + msg = c.message(alice2, chat, "before-redis-restart")["json"] + compose("restart", "redis") + wait_ready(c) + me = c.call("GET", "/api/v1/users/me", actor=alice2, schema="User")["json"] + check(me["id"] == alice2["id"], "session lost after Redis restart despite AOF") + page = c.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=bob, + schema="MessagePage", + )["json"] + check(any(m["id"] == msg["id"] for m in page["items"]), "history not rebuilt after cache restart") + print(json.dumps({"redis_restart": "session+history ok", "db_or_redis_down": "503"}, ensure_ascii=False)) + print("PASS: infra unavailability and redis restart") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 00000000..9117863d --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Student checks for grades 4 and 5 (track 1: cache consistency).""" +import base64 +import json +import os +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run # noqa: E402 + + +def _basic(username, password): + return "Basic " + base64.b64encode((username + ":" + password).encode()).decode() + + +def lab_metrics(c: Client): + return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode()) + + +def lab_flags(c: Client, body): + return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode()) + + +def test_wrong_password_and_x_user_id(c: Client): + actor = c.user("neg") + bad = _basic(actor["registration"]["username"], "wrong-password-xx") + c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad}) + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": actor["id"]}) + c.call("GET", "/api/v1/users/me", 401) + + +def test_logout(c: Client): + actor = c.user("out") + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor) + c.call("GET", "/api/v1/users/me", 401, actor=actor) + + +def test_register_race(c: Client): + username = "r_" + uuid.uuid4().hex[:12] + body = { + "username": username, + "display_name": "Race", + "password": "Example-only-not-a-real-secret-123", + } + results = [] + + def once(): + local = Client(c.base) + try: + results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]) + except Failure as exc: + results.append(str(exc)) + + threads = [threading.Thread(target=once) for _ in range(16)] + for t in threads: + t.start() + for t in threads: + t.join() + created = results.count(201) + conflicted = results.count(409) + check(created == 1, "username race: expected exactly one 201, got %s" % results) + check(conflicted == 15, "username race: expected fifteen 409, got %s" % results) + + +def test_chat_rollback(c: Client): + alice = c.user("rb") + before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + missing = str(uuid.uuid4()) + c.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]}, + ) + after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + check(len(after) == len(before), "composite chat create leaked a row after rollback") + check(all(x["title"] != "rollback-should-not-exist" for x in after), "rolled-back title became visible") + + +def test_read_after_write(c: Client): + alice, bob, _eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + lab_flags(c, {"invalidate": True, "singleflight": True, "reset_sql": True}) + first = c.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"] + page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + check(any(m["id"] == first["id"] for m in page["items"]), "read-after-write lost the new message") + + +def test_stale_without_invalidation_then_fixed(c: Client): + alice, bob, _eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50" + c.call("GET", path, actor=alice, schema="MessagePage") + lab_flags(c, {"invalidate": False, "singleflight": True}) + stale_text = "stale-probe-" + uuid.uuid4().hex + c.message(alice, chat, stale_text) + page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + stale = all(m["text"] != stale_text for m in page["items"]) + check(stale, "expected a stale cache read when invalidation is disabled") + lab_flags(c, {"invalidate": True}) + fresh_text = "fresh-probe-" + uuid.uuid4().hex + c.message(alice, chat, fresh_text) + page2 = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + check(any(m["text"] == fresh_text for m in page2["items"]), "invalidation did not restore read-after-write") + + +def test_stampede(c: Client): + alice, _bob, _eve, chat = c.bootstrap() + c.message(alice, chat, "stampede-seed") + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50" + + def burst(n=24): + barrier = threading.Barrier(n) + + def one(_i): + local = Client(c.base) + barrier.wait() + return local.call("GET", path, actor=alice, schema="MessagePage") + + with ThreadPoolExecutor(max_workers=n) as pool: + list(pool.map(one, range(n))) + + lab_flags(c, {"singleflight": False, "invalidate": True, "reset_sql": True}) + c.message(alice, chat, "miss-" + uuid.uuid4().hex) + lab_flags(c, {"reset_sql": True, "singleflight": False}) + burst() + unprotected = lab_metrics(c)["history_sql"] + + lab_flags(c, {"singleflight": True, "reset_sql": True}) + c.message(alice, chat, "miss2-" + uuid.uuid4().hex) + lab_flags(c, {"reset_sql": True, "singleflight": True}) + burst() + protected = lab_metrics(c)["history_sql"] + print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected}, ensure_ascii=False)) + check(unprotected >= 8, "unprotected stampede did not generate enough SQL") + check(protected < unprotected, "singleflight did not reduce SQL during stampede") + check(protected <= 4, "singleflight still issued too many history SQL queries") + + +def test_expiry(c: Client): + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + check(ttl <= 3, "expiry test requires SESSION_TTL_SECONDS<=3 (recreate api with short TTL)") + actor = c.user("ttl") + c.call("GET", "/api/v1/users/me", actor=actor, schema="User") + time.sleep(ttl + 1.5) + c.call("GET", "/api/v1/users/me", 401, actor=actor) + + +def main(): + base = os.environ.get("BASE_URL", "http://localhost:8080") + c = Client(base) + c.call("GET", "/health/live", schema="Health") + c.call("GET", "/health/ready", schema="Health") + tests = [ + test_wrong_password_and_x_user_id, + test_logout, + test_register_race, + test_chat_rollback, + test_read_after_write, + test_stale_without_invalidation_then_fixed, + test_stampede, + ] + if os.environ.get("RUN_EXPIRY") == "1": + tests.append(test_expiry) + try: + for fn in tests: + fn(c) + print("PASS", fn.__name__) + finally: + try: + lab_flags(c, {"invalidate": True, "singleflight": True}) + except Exception: + pass + print("PASS: student grade 4/5 HTTP checks") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From bd0b33e273ca3279f915d15d1123e8adb863d78c Mon Sep 17 00:00:00 2001 From: David Date: Sat, 7 Nov 2026 11:15:20 +0300 Subject: [PATCH 2/3] Revert "finished" This reverts commit 7329d94740039d52a67f7cdd9032faba3174c2c5. --- .env.example | 10 +- .github/mtusi2026lab2.lnk | Bin 1250 -> 0 bytes .github/workflows/contracts.yml | 26 +-- Dockerfile | 24 -- Makefile | 14 +- compose.yaml | 78 ------- migrations/001_init.sql | 51 ----- requirements.txt | 8 - scripts/restart.py | 21 -- scripts/restart.sh | 9 - scripts/run_expiry.py | 51 ----- src/app/__init__.py | 1 - src/app/cache.py | 133 ----------- src/app/config.py | 33 --- src/app/context.py | 8 - src/app/cursors.py | 35 --- src/app/db.py | 98 --------- src/app/errors.py | 25 --- src/app/main.py | 76 ------- src/app/migrate.py | 38 ---- src/app/passwords.py | 23 -- src/app/rediscli.py | 44 ---- src/app/routes.py | 378 -------------------------------- src/app/serialize.py | 31 --- src/app/sessions.py | 56 ----- src/app/timeutil.py | 12 - src/app/tools/__init__.py | 0 src/app/tools/explain.py | 85 ------- student-tests/expiry.py | 22 -- student-tests/infra.py | 92 -------- student-tests/run.py | 184 ---------------- 31 files changed, 4 insertions(+), 1662 deletions(-) delete mode 100644 .github/mtusi2026lab2.lnk delete mode 100644 Dockerfile delete mode 100644 compose.yaml delete mode 100644 migrations/001_init.sql delete mode 100644 requirements.txt delete mode 100644 scripts/restart.py delete mode 100644 scripts/restart.sh delete mode 100644 scripts/run_expiry.py delete mode 100644 src/app/__init__.py delete mode 100644 src/app/cache.py delete mode 100644 src/app/config.py delete mode 100644 src/app/context.py delete mode 100644 src/app/cursors.py delete mode 100644 src/app/db.py delete mode 100644 src/app/errors.py delete mode 100644 src/app/main.py delete mode 100644 src/app/migrate.py delete mode 100644 src/app/passwords.py delete mode 100644 src/app/rediscli.py delete mode 100644 src/app/routes.py delete mode 100644 src/app/serialize.py delete mode 100644 src/app/sessions.py delete mode 100644 src/app/timeutil.py delete mode 100644 src/app/tools/__init__.py delete mode 100644 src/app/tools/explain.py delete mode 100644 student-tests/expiry.py delete mode 100644 student-tests/infra.py delete mode 100644 student-tests/run.py diff --git a/.env.example b/.env.example index e065c373..714c8076 100644 --- a/.env.example +++ b/.env.example @@ -4,16 +4,8 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -CACHE_TTL_SECONDS=60 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=redis +SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url - -# Compose interpolates these. Synthetic local values only; replace after copy. -POSTGRES_PASSWORD=lab2-dev-only-change-me -CURSOR_SIGNING_KEY=lab2-dev-cursor-key-change-me-32bytes -# Application also accepts DATABASE_URL / REDIS_URL (compose injects them). -DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger -REDIS_URL=redis://redis:6379/0 diff --git a/.github/mtusi2026lab2.lnk b/.github/mtusi2026lab2.lnk deleted file mode 100644 index 5cfc18e4e1271ef0fda02d8a455297549470a6d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1250 zcmai!Ur1AN6vxl(KB$$=Tq0s^Scy{FocL!`OQ-0pWr7Koci`OIpPX%Yr}&bik%XiW zL=XkhA4`IQgd~JkK12jXNDoDpf!IUCa`;p)Qr~murfJk2=iK}Io%8$r&i$VIo0*6d zsswnVDcMuq1xh3(j{|LGn@`Uwp45$0@%^)ORvybi#1+x}xWA$>klJK;{p`{Hk7nb; zyg>Ns`4`nW$jhO~(^TYTO-8iar#~siLc3i9)>%bXD5WSFWEC}0q0DKr%%4zFx4+ls zrV2_T0V^9UU)vlOAy|NntqYvCQ_fv=fr~!sCO`F(jog@4(a9B_!;%(MlFRLMdAwFT zkulDUfSD#UOTI9vXcZ?vO&r#f3p6LZhrDDZET;@=jFT5Z_ENV z{EJmhUq=l@Zh>Z{4#G0jX^+#EK|6Z9rMW>jfRH&(Tn2GjChD&-taTkR`#_>1`tA~{ z)sYrfvSyDA~3N{e3oPG0n99px(rJ^WdA{w|-UI z+@87ChV=JKsh74qugnZjF8wdQ54t9Se-rx@Ha9hY*M??lXsCYd=XBbrG~JF+y06e6P@5xtcx(9@h35jbOM4$j>NAyhmJ-^BuP@wZ^^-Y+3rvCit9?5V-v*;9cDIN0 zd=mMr2s~y2H%*OF4+#c%E>lCbc+#U(tx$pDiI~DW$&~mx#yBS8-BY&Q%FAz4C*IN2 zu@4#3EaEO+5EC&fTnkCmNj>m?N>z(liMntEz2rk|^)PM}NQ^fmmiPA$mv@ggkVPC% s@2-6?;xmnzstQIc`-k~2z{LqB*m#Z*di64FJQs47pJ}{T$Mdo4H}A>^d;kCd diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 7e020ea6..11cf289f 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,27 +12,5 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python - application: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - run: cp .env.example .env - - run: docker compose --env-file .env up -d --build - - name: Wait until API is ready - run: | - for i in $(seq 1 60); do - if curl -fsS http://127.0.0.1:8080/health/ready; then exit 0; fi - sleep 2 - done - docker compose --env-file .env logs - exit 1 - - run: make test PYTHON=python - - run: chmod +x scripts/restart.sh - - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python - - run: python student-tests/run.py - - run: python student-tests/infra.py - - run: python scripts/run_expiry.py - - run: docker compose --env-file .env exec -T api python -m app.tools.explain +# This job validates the skeleton, not a student solution. +# Add build, readiness and make test in a separate application job. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 1b537886..00000000 --- a/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM python:3.12.8-slim-bookworm - -RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser \ - && apt-get update \ - && apt-get install -y --no-install-recommends curl \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt - -COPY src/app /app/app -COPY migrations /app/migrations - -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PYTHONPATH=/app - -USER appuser -EXPOSE 8080 -HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \ - CMD curl -fsS http://127.0.0.1:8080/health/ready || exit 1 - -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Makefile b/Makefile index bc3e2382..9ecb223e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-student test-infra test-expiry explain +.PHONY: help check validate test load test-persistence help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,15 +24,3 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) - -test-student: - $(PYTHON) student-tests/run.py - -test-infra: - $(PYTHON) student-tests/infra.py - -test-expiry: - $(PYTHON) scripts/run_expiry.py - -explain: - docker compose --env-file .env exec -T api python -m app.tools.explain diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index 0addf266..00000000 --- a/compose.yaml +++ /dev/null @@ -1,78 +0,0 @@ -services: - api: - build: - context: . - dockerfile: Dockerfile - image: mtusi-lab2-api:1.2.0 - ports: - - "8080:8080" - environment: - APP_HOST: 0.0.0.0 - APP_PORT: "8080" - APP_ENV: lab - SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} - CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60} - DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger - REDIS_URL: redis://redis:6379/0 - CURSOR_SIGNING_KEY: ${CURSOR_SIGNING_KEY} - DB_POOL_MIN: "1" - DB_POOL_MAX: "10" - DB_COMMAND_TIMEOUT: "5" - DB_CONNECT_TIMEOUT: "5" - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - restart: unless-stopped - mem_limit: 512m - cpus: "1.0" - healthcheck: - test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"] - interval: 5s - timeout: 3s - retries: 12 - start_period: 20s - - postgres: - image: postgres:16.6-alpine - environment: - POSTGRES_USER: messenger - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: messenger - volumes: - - pg_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] - interval: 3s - timeout: 3s - retries: 20 - restart: unless-stopped - mem_limit: 512m - cpus: "1.0" - - redis: - image: redis:7.4.2-alpine - command: - - redis-server - - --appendonly - - "yes" - - --appendfsync - - everysec - - --save - - "60" - - "1" - volumes: - - redis_data:/data - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 3s - timeout: 3s - retries: 20 - restart: unless-stopped - mem_limit: 128m - cpus: "0.50" - -volumes: - pg_data: - redis_data: diff --git a/migrations/001_init.sql b/migrations/001_init.sql deleted file mode 100644 index 12b88622..00000000 --- a/migrations/001_init.sql +++ /dev/null @@ -1,51 +0,0 @@ --- Lab2 schema. PostgreSQL is the source of truth for users, chats, members, messages. --- Transaction boundary: create-chat inserts the chat row and all members in one transaction. --- Sessions live in Redis (AOF); this database does not store session tokens. - -CREATE TABLE IF NOT EXISTS schema_migrations ( - filename TEXT PRIMARY KEY, - applied_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE TABLE users ( - id UUID PRIMARY KEY, - username TEXT NOT NULL, - display_name TEXT NOT NULL, - password_hash TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), - CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) -); - -CREATE UNIQUE INDEX users_username_key ON users (username); - -CREATE TABLE chats ( - id UUID PRIMARY KEY, - title TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) -); - -CREATE TABLE chat_members ( - chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, - user_id UUID NOT NULL REFERENCES users (id), - PRIMARY KEY (chat_id, user_id) -); - -CREATE INDEX chat_members_user_id_idx ON chat_members (user_id); - --- List chats of a member ordered by (created_at, id). -CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); - -CREATE TABLE messages ( - id UUID PRIMARY KEY, - chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, - sender_id UUID NOT NULL REFERENCES users (id), - text TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) -); - --- Keyset pagination: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n --- This composite index avoids a sequential scan of the whole history per page. -CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d5ab2583..00000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -fastapi==0.115.6 -uvicorn==0.32.1 -asyncpg==0.30.0 -redis==5.2.1 -argon2-cffi==23.1.0 -starlette==0.41.3 -pydantic==2.10.3 -python-multipart==0.0.20 diff --git a/scripts/restart.py b/scripts/restart.py deleted file mode 100644 index fb4c78d8..00000000 --- a/scripts/restart.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -"""Restart API, PostgreSQL and Redis without removing named volumes.""" -from pathlib import Path -import subprocess -import sys - -ROOT = Path(__file__).resolve().parents[1] - - -def main() -> int: - env_file = ROOT / ".env" - cmd = ["docker", "compose"] - if env_file.is_file(): - cmd.extend(["--env-file", str(env_file)]) - cmd.extend(["restart", "postgres", "redis", "api"]) - proc = subprocess.run(cmd, cwd=ROOT) - return proc.returncode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh deleted file mode 100644 index 81d27153..00000000 --- a/scripts/restart.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -eu -ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) -cd "$ROOT" -if [ -f .env ]; then - docker compose --env-file .env restart postgres redis api -else - docker compose restart postgres redis api -fi diff --git a/scripts/run_expiry.py b/scripts/run_expiry.py deleted file mode 100644 index a5f51c4c..00000000 --- a/scripts/run_expiry.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL.""" -import os -import subprocess -import sys -import time -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] - - -def compose(args, extra_env=None): - env = os.environ.copy() - if extra_env: - env.update(extra_env) - cmd = ["docker", "compose"] - if (ROOT / ".env").is_file(): - cmd.extend(["--env-file", str(ROOT / ".env")]) - cmd.extend(args) - proc = subprocess.run(cmd, cwd=ROOT, env=env) - if proc.returncode != 0: - raise SystemExit(proc.returncode) - - -def wait_ready(): - import urllib.request - - deadline = time.monotonic() + 90 - while time.monotonic() < deadline: - try: - urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) - return - except Exception: - time.sleep(0.5) - raise SystemExit("API did not become ready") - - -def main(): - compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"}) - wait_ready() - env = os.environ.copy() - env["SESSION_TTL_SECONDS"] = "2" - env["RUN_EXPIRY"] = "1" - proc = subprocess.run([sys.executable, str(ROOT / "student-tests" / "expiry.py")], cwd=ROOT, env=env) - compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"}) - wait_ready() - raise SystemExit(proc.returncode) - - -if __name__ == "__main__": - main() diff --git a/src/app/__init__.py b/src/app/__init__.py deleted file mode 100644 index 045965c1..00000000 --- a/src/app/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""MTUSI messenger API, laboratory 2.""" diff --git a/src/app/cache.py b/src/app/cache.py deleted file mode 100644 index 2dc8bd5f..00000000 --- a/src/app/cache.py +++ /dev/null @@ -1,133 +0,0 @@ -import asyncio -import json -from collections.abc import Awaitable, Callable -from typing import TypeVar - -from . import rediscli -from .config import settings -from .errors import AppError - -T = TypeVar("T") - -CFG_SINGLEFLIGHT = "cfg:singleflight" -CFG_INVALIDATE = "cfg:invalidate" -SQL_HISTORY = "metrics:sql:messages_select" - - -class SingleFlight: - def __init__(self) -> None: - self._lock = asyncio.Lock() - self._inflight: dict[str, asyncio.Task] = {} - - async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T: - async with self._lock: - task = self._inflight.get(key) - if task is None: - task = asyncio.create_task(factory()) - self._inflight[key] = task - owner = True - else: - owner = False - try: - return await asyncio.shield(task) - finally: - if owner: - async with self._lock: - self._inflight.pop(key, None) - - -_sf = SingleFlight() - - -async def _flag(name: str, default: bool = True) -> bool: - if not settings.lab_mode: - return default - try: - raw = await rediscli.require().get(name) - except Exception: - return default - if raw is None: - return default - return raw not in {"0", "false", "off"} - - -async def singleflight_enabled() -> bool: - return await _flag(CFG_SINGLEFLIGHT, True) - - -async def invalidate_enabled() -> bool: - return await _flag(CFG_INVALIDATE, True) - - -async def incr_history_sql() -> None: - try: - await rediscli.require().incr(SQL_HISTORY) - except Exception: - pass - - -async def history_sql_count() -> int: - try: - raw = await rediscli.require().get(SQL_HISTORY) - return int(raw or 0) - except Exception: - return 0 - - -async def reset_history_sql_count() -> None: - try: - await rediscli.require().set(SQL_HISTORY, 0) - except Exception: - pass - - -async def set_flags(*, singleflight: bool | None = None, invalidate: bool | None = None) -> dict: - r = rediscli.require() - if singleflight is not None: - await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0") - if invalidate is not None: - await r.set(CFG_INVALIDATE, "1" if invalidate else "0") - return { - "singleflight": await singleflight_enabled(), - "invalidate": await invalidate_enabled(), - "history_sql": await history_sql_count(), - } - - -def _page_key(chat_id: str, version: str, cursor: str, limit: int) -> str: - return f"hist:{chat_id}:{version}:{cursor}:{limit}" - - -async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict: - r = rediscli.require() - try: - version = await r.get("histver:" + chat_id) or "0" - key = _page_key(chat_id, version, cursor, limit) - cached = await r.get(key) - if cached: - return json.loads(cached) - - async def load_and_store() -> dict: - page = await loader() - try: - await r.set(key, json.dumps(page, ensure_ascii=False), ex=settings.cache_ttl_seconds) - except Exception: - pass - return page - - if await singleflight_enabled(): - return await _sf.do(key, load_and_store) - return await load_and_store() - except AppError: - raise - except Exception: - return await loader() - - -async def invalidate_history(chat_id: str) -> None: - if not await invalidate_enabled(): - return - try: - await rediscli.require().incr("histver:" + chat_id) - except Exception: - pass diff --git a/src/app/config.py b/src/app/config.py deleted file mode 100644 index f7f4ed52..00000000 --- a/src/app/config.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -from pathlib import Path - - -def _file_or_env(name: str, default: str = "") -> str: - path = os.environ.get(name + "_FILE") - if path: - return Path(path).read_text(encoding="utf-8").strip() - return os.environ.get(name, default) - - -class Settings: - def __init__(self) -> None: - self.app_host = os.environ.get("APP_HOST", "0.0.0.0") - self.app_port = int(os.environ.get("APP_PORT", "8080")) - self.app_env = os.environ.get("APP_ENV", "lab") - self.session_ttl_seconds = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) - self.cache_ttl_seconds = int(os.environ.get("CACHE_TTL_SECONDS", "60")) - self.database_url = _file_or_env("DATABASE_URL") - self.redis_url = _file_or_env("REDIS_URL", "redis://redis:6379/0") - self.cursor_signing_key = _file_or_env("CURSOR_SIGNING_KEY", "lab2-dev-cursor-key-change-me") - self.db_pool_min = int(os.environ.get("DB_POOL_MIN", "1")) - self.db_pool_max = int(os.environ.get("DB_POOL_MAX", "10")) - self.db_command_timeout = float(os.environ.get("DB_COMMAND_TIMEOUT", "5")) - self.db_connect_timeout = float(os.environ.get("DB_CONNECT_TIMEOUT", "5")) - self.redis_socket_timeout = float(os.environ.get("REDIS_SOCKET_TIMEOUT", "2")) - - @property - def lab_mode(self) -> bool: - return self.app_env == "lab" - - -settings = Settings() diff --git a/src/app/context.py b/src/app/context.py deleted file mode 100644 index 632762bd..00000000 --- a/src/app/context.py +++ /dev/null @@ -1,8 +0,0 @@ -from contextvars import ContextVar - -request_id_var: ContextVar[str] = ContextVar("request_id", default="") -sql_count_var: ContextVar[int] = ContextVar("sql_count", default=0) - - -def bump_sql(n: int = 1) -> None: - sql_count_var.set(sql_count_var.get() + n) diff --git a/src/app/cursors.py b/src/app/cursors.py deleted file mode 100644 index c1715ce1..00000000 --- a/src/app/cursors.py +++ /dev/null @@ -1,35 +0,0 @@ -import base64 -import hashlib -import hmac -import json - -from .config import settings -from .errors import AppError - - -def _sign(payload: str) -> str: - key = settings.cursor_signing_key.encode("utf-8") - return hmac.new(key, payload.encode("utf-8"), hashlib.sha256).hexdigest()[:24] - - -def encode_cursor(data: dict) -> str: - payload = json.dumps(data, separators=(",", ":"), ensure_ascii=False) - raw = payload + "." + _sign(payload) - return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") - - -def decode_cursor(token: str, kind: str, chat_id: str | None = None) -> dict: - try: - padded = token + "=" * (-len(token) % 4) - raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8") - payload, _, sig = raw.partition(".") - if not sig or not hmac.compare_digest(sig, _sign(payload)): - raise ValueError("bad signature") - data = json.loads(payload) - if data.get("k") != kind: - raise ValueError("wrong kind") - if chat_id is not None and data.get("c") != chat_id: - raise ValueError("wrong chat") - return data - except Exception: - raise AppError(400, "invalid_request", "Некорректный cursor") from None diff --git a/src/app/db.py b/src/app/db.py deleted file mode 100644 index 3f4e2d39..00000000 --- a/src/app/db.py +++ /dev/null @@ -1,98 +0,0 @@ -import asyncpg - -from .config import settings -from .context import bump_sql -from .errors import AppError - -pool: asyncpg.Pool | None = None - - -async def connect() -> asyncpg.Pool: - global pool - if not settings.database_url: - raise RuntimeError("DATABASE_URL is not set") - pool = await asyncpg.create_pool( - dsn=settings.database_url, - min_size=settings.db_pool_min, - max_size=settings.db_pool_max, - command_timeout=settings.db_command_timeout, - timeout=settings.db_connect_timeout, - ) - return pool - - -async def close() -> None: - global pool - if pool is not None: - await pool.close() - pool = None - - -def require_pool() -> asyncpg.Pool: - if pool is None: - raise AppError(503, "unavailable", "Сервис временно недоступен") - return pool - - -async def ping() -> None: - async with require_pool().acquire() as conn: - bump_sql() - await conn.execute("SELECT 1") - - -_INTEGRITY = ( - asyncpg.UniqueViolationError, - asyncpg.ForeignKeyViolationError, - asyncpg.CheckViolationError, -) - - -async def fetch(sql: str, *args, count: bool = True): - try: - async with require_pool().acquire() as conn: - if count: - bump_sql() - return await conn.fetch(sql, *args) - except AppError: - raise - except _INTEGRITY: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - - -async def fetchrow(sql: str, *args, count: bool = True): - try: - async with require_pool().acquire() as conn: - if count: - bump_sql() - return await conn.fetchrow(sql, *args) - except AppError: - raise - except _INTEGRITY: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - - -async def execute(sql: str, *args, count: bool = True): - try: - async with require_pool().acquire() as conn: - if count: - bump_sql() - return await conn.execute(sql, *args) - except AppError: - raise - except _INTEGRITY: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - - -async def transaction_connection(): - try: - return require_pool().acquire() - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None diff --git a/src/app/errors.py b/src/app/errors.py deleted file mode 100644 index f41964ce..00000000 --- a/src/app/errors.py +++ /dev/null @@ -1,25 +0,0 @@ -from fastapi.responses import JSONResponse - - -class AppError(Exception): - def __init__(self, status: int, code: str, message: str, www_authenticate: str | None = None) -> None: - super().__init__(message) - self.status = status - self.code = code - self.message = message - self.www_authenticate = www_authenticate - - -def error_response(status: int, code: str, message: str, request_id: str, www_authenticate: str | None = None) -> JSONResponse: - headers = {"X-Request-Id": request_id} - if www_authenticate: - headers["WWW-Authenticate"] = www_authenticate - return JSONResponse( - status_code=status, - content={"error": {"code": code, "message": message, "request_id": request_id}}, - headers=headers, - ) - - -BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' -BEARER_CHALLENGE = 'Bearer realm="messenger"' diff --git a/src/app/main.py b/src/app/main.py deleted file mode 100644 index ad54f3f5..00000000 --- a/src/app/main.py +++ /dev/null @@ -1,76 +0,0 @@ -from contextlib import asynccontextmanager -import uuid - -from fastapi import FastAPI, Request -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse -from starlette.exceptions import HTTPException as StarletteHTTPException - -from . import db, rediscli -from .config import settings -from .context import request_id_var, sql_count_var -from .errors import AppError, error_response -from .migrate import apply_migrations -from .routes import api, health - - -@asynccontextmanager -async def lifespan(_app: FastAPI): - await db.connect() - await apply_migrations(db.require_pool()) - await rediscli.connect() - yield - await rediscli.close() - await db.close() - - -app = FastAPI(title="MTUSI Messenger", version="1.2.0", lifespan=lifespan, redirect_slashes=False) -app.include_router(health) -app.include_router(api, prefix="/api/v1") - - -@app.middleware("http") -async def request_context(request: Request, call_next): - rid = request.headers.get("X-Request-Id") or str(uuid.uuid4()) - request.state.request_id = rid - token_rid = request_id_var.set(rid) - token_sql = sql_count_var.set(0) - sql_count = 0 - try: - response = await call_next(request) - sql_count = sql_count_var.get() - finally: - request_id_var.reset(token_rid) - sql_count_var.reset(token_sql) - response.headers["X-Request-Id"] = rid - if settings.lab_mode: - response.headers["X-Sql-Count"] = str(sql_count) - return response - - -@app.exception_handler(AppError) -async def handle_app_error(request: Request, exc: AppError): - rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) - return error_response(exc.status, exc.code, exc.message, rid, exc.www_authenticate) - - -@app.exception_handler(RequestValidationError) -async def handle_validation(request: Request, _exc: RequestValidationError): - rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) - return error_response(400, "invalid_request", "Некорректный запрос", rid) - - -@app.exception_handler(StarletteHTTPException) -async def handle_http(request: Request, exc: StarletteHTTPException): - rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) - if exc.status_code == 404: - return error_response(404, "not_found", "Объект не найден", rid) - if exc.status_code == 405: - return error_response(400, "invalid_request", "Некорректный запрос", rid) - return error_response(exc.status_code, "internal_error", "Внутренняя ошибка", rid) - - -@app.exception_handler(Exception) -async def handle_unexpected(request: Request, _exc: Exception): - rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) - return error_response(500, "internal_error", "Внутренняя ошибка", rid) diff --git a/src/app/migrate.py b/src/app/migrate.py deleted file mode 100644 index 9215e7c9..00000000 --- a/src/app/migrate.py +++ /dev/null @@ -1,38 +0,0 @@ -from pathlib import Path - -import asyncpg - - -def _migrations_dir() -> Path: - here = Path(__file__).resolve().parent - for candidate in (here.parent.parent / "migrations", Path("/app/migrations")): - if candidate.is_dir(): - return candidate - raise RuntimeError("migrations directory not found") - - -MIGRATIONS = _migrations_dir() - - -async def apply_migrations(pool: asyncpg.Pool) -> None: - async with pool.acquire() as conn: - await conn.execute( - """ - CREATE TABLE IF NOT EXISTS schema_migrations ( - filename TEXT PRIMARY KEY, - applied_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - """ - ) - applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")} - files = sorted(p for p in MIGRATIONS.glob("*.sql") if p.is_file()) - for path in files: - if path.name in applied: - continue - sql = path.read_text(encoding="utf-8") - async with conn.transaction(): - await conn.execute(sql) - await conn.execute( - "INSERT INTO schema_migrations(filename) VALUES($1)", - path.name, - ) diff --git a/src/app/passwords.py b/src/app/passwords.py deleted file mode 100644 index 8431b234..00000000 --- a/src/app/passwords.py +++ /dev/null @@ -1,23 +0,0 @@ -from argon2 import PasswordHasher -from argon2.exceptions import InvalidHashError, VerifyMismatchError - -hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1) -_dummy_hash = None - - -def hash_password(password: str) -> str: - return hasher.hash(password) - - -def dummy_hash() -> str: - global _dummy_hash - if _dummy_hash is None: - _dummy_hash = hash_password("timing-dummy-password") - return _dummy_hash - - -def verify_password(password_hash: str, password: str) -> bool: - try: - return hasher.verify(password_hash, password) - except (VerifyMismatchError, InvalidHashError, ValueError, TypeError): - return False diff --git a/src/app/rediscli.py b/src/app/rediscli.py deleted file mode 100644 index e38c6b5b..00000000 --- a/src/app/rediscli.py +++ /dev/null @@ -1,44 +0,0 @@ -import redis.asyncio as redis - -from .config import settings -from .errors import AppError - -client: redis.Redis | None = None - - -async def connect() -> redis.Redis: - global client - client = redis.from_url( - settings.redis_url, - socket_connect_timeout=settings.redis_socket_timeout, - socket_timeout=settings.redis_socket_timeout, - decode_responses=True, - ) - await client.ping() - return client - - -async def close() -> None: - global client - if client is not None: - await client.close() - client = None - - -def require() -> redis.Redis: - if client is None: - raise AppError(503, "unavailable", "Сервис временно недоступен") - return client - - -async def ping() -> None: - try: - await require().ping() - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - - -def wrap_redis_errors(exc: Exception) -> AppError: - return AppError(503, "unavailable", "Сервис временно недоступен") diff --git a/src/app/routes.py b/src/app/routes.py deleted file mode 100644 index e80b9de8..00000000 --- a/src/app/routes.py +++ /dev/null @@ -1,378 +0,0 @@ -import base64 -import binascii -from datetime import datetime, timezone -from uuid import UUID, uuid4 - -import asyncpg -from fastapi import APIRouter, Request -from fastapi.responses import JSONResponse, Response -from pydantic import BaseModel, ConfigDict, Field - -from . import cache, db, rediscli, sessions -from .cursors import decode_cursor, encode_cursor -from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError -from .config import settings -from .passwords import dummy_hash, hash_password, verify_password -from .serialize import chat_json, message_json, user_json -from .timeutil import fmt_dt - -api = APIRouter() -health = APIRouter() - - -class CreateUser(BaseModel): - model_config = ConfigDict(extra="forbid") - username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") - display_name: str = Field(min_length=1, max_length=100) - password: str = Field(min_length=12, max_length=128) - - -class CreateChat(BaseModel): - model_config = ConfigDict(extra="forbid") - title: str = Field(min_length=1, max_length=100) - member_ids: list[UUID] = Field(min_length=1, max_length=100) - - -class CreateMessage(BaseModel): - model_config = ConfigDict(extra="forbid") - text: str = Field(min_length=1, max_length=4000) - - -def _blank(value: str) -> bool: - return value.strip() == "" - - -def _limit(raw: str | None) -> int: - if raw is None or raw == "": - return 50 - try: - value = int(raw) - except ValueError: - raise AppError(400, "invalid_request", "Некорректный limit") from None - if value < 1 or value > 100: - raise AppError(400, "invalid_request", "Некорректный limit") - return value - - -async def current_user(request: Request) -> asyncpg.Record: - header = request.headers.get("Authorization") or "" - scheme, _, rest = header.partition(" ") - if scheme.lower() != "bearer" or not rest.strip(): - raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) - user_id = await sessions.resolve(rest.strip()) - row = await db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id) - if row is None: - raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) - return row - - -async def member_ids_of(chat_id: UUID) -> list[UUID]: - rows = await db.fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id) - return [r["user_id"] for r in rows] - - -async def load_chat_for_member(chat_id: UUID, user_id: UUID) -> asyncpg.Record: - row = await db.fetchrow( - """ - SELECT c.id, c.title, c.created_at - FROM chats c - JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 - WHERE c.id = $1 - """, - chat_id, - user_id, - ) - if row is None: - raise AppError(404, "not_found", "Объект не найден") - return row - - -@health.get("/health/live") -async def live() -> JSONResponse: - return JSONResponse({"status": "ok"}) - - -@health.get("/health/ready") -async def ready() -> JSONResponse: - await db.ping() - await rediscli.ping() - return JSONResponse({"status": "ok"}) - - -@api.post("/users") -async def register(body: CreateUser) -> JSONResponse: - if _blank(body.display_name) or _blank(body.password) or _blank(body.username): - raise AppError(400, "invalid_request", "Некорректные поля") - user_id = uuid4() - created = datetime.now(timezone.utc) - try: - await db.execute( - """ - INSERT INTO users (id, username, display_name, password_hash, created_at) - VALUES ($1, $2, $3, $4, $5) - """, - user_id, - body.username, - body.display_name, - hash_password(body.password), - created, - ) - except asyncpg.UniqueViolationError: - raise AppError(409, "conflict", "Имя пользователя занято") from None - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - return JSONResponse( - user_json({"id": user_id, "username": body.username, "display_name": body.display_name, "created_at": created}), - status_code=201, - ) - - -@api.get("/users/me") -async def me(request: Request) -> JSONResponse: - row = await current_user(request) - return JSONResponse(user_json(row)) - - -@api.post("/auth/sessions") -async def login(request: Request) -> JSONResponse: - header = request.headers.get("Authorization") or "" - scheme, _, rest = header.partition(" ") - if scheme.lower() != "basic" or not rest.strip(): - raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) - try: - decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8") - except (binascii.Error, UnicodeError, ValueError): - raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) from None - username, sep, password = decoded.partition(":") - if not sep: - raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) - row = await db.fetchrow( - "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1", - username, - ) - password_hash = row["password_hash"] if row else dummy_hash() - if row is None or not verify_password(password_hash, password): - raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) - token, expires_at = await sessions.create(row["id"]) - return JSONResponse( - { - "token": token, - "token_type": "Bearer", - "expires_at": expires_at, - "user": user_json(row), - }, - status_code=201, - ) - - -@api.delete("/auth/sessions/current") -async def logout(request: Request) -> Response: - header = request.headers.get("Authorization") or "" - scheme, _, rest = header.partition(" ") - if scheme.lower() != "bearer" or not rest.strip(): - raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) - await sessions.revoke(rest.strip()) - return Response(status_code=204) - - -@api.post("/chats") -async def create_chat(request: Request, body: CreateChat) -> JSONResponse: - if _blank(body.title): - raise AppError(400, "invalid_request", "Некорректные поля") - if len(body.member_ids) != len(set(body.member_ids)): - raise AppError(400, "invalid_request", "Некорректный список участников") - user = await current_user(request) - if user["id"] not in body.member_ids: - raise AppError(400, "invalid_request", "Некорректный список участников") - chat_id = uuid4() - created = datetime.now(timezone.utc) - try: - async with db.require_pool().acquire() as conn: - async with conn.transaction(): - existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", body.member_ids) - if len(existing) != len(body.member_ids): - raise AppError(400, "invalid_request", "Некорректный список участников") - await conn.execute( - "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", - chat_id, - body.title, - created, - ) - await conn.executemany( - "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", - [(chat_id, mid) for mid in body.member_ids], - ) - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - return JSONResponse(chat_json({"id": chat_id, "title": body.title, "created_at": created}, body.member_ids), status_code=201) - - -@api.get("/chats") -async def list_chats(request: Request) -> JSONResponse: - user = await current_user(request) - limit = _limit(request.query_params.get("limit")) - cursor = request.query_params.get("cursor") - args: list = [user["id"]] - clause = "" - if cursor: - data = decode_cursor(cursor, "chat") - clause = "AND (c.created_at, c.id) > ($2, $3)" - args.extend([datetime.fromisoformat(data["t"].replace("Z", "+00:00")), UUID(data["i"])]) - rows = await db.fetch( - f""" - SELECT c.id, c.title, c.created_at - FROM chats c - JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1 - WHERE 1=1 {clause} - ORDER BY c.created_at, c.id - LIMIT {limit + 1} - """, - *args, - ) - page = rows[:limit] - items = [] - for row in page: - items.append(chat_json(row, await member_ids_of(row["id"]))) - next_cursor = None - if len(rows) > limit: - last = page[-1] - next_cursor = encode_cursor({"k": "chat", "t": fmt_dt(last["created_at"]), "i": str(last["id"])}) - return JSONResponse({"items": items, "next_cursor": next_cursor}) - - -@api.get("/chats/{chat_id}") -async def get_chat(request: Request, chat_id: UUID) -> JSONResponse: - user = await current_user(request) - row = await load_chat_for_member(chat_id, user["id"]) - return JSONResponse(chat_json(row, await member_ids_of(chat_id))) - - -async def _load_messages(chat_id: UUID, after_ts: datetime | None, after_id: UUID | None, limit: int) -> dict: - await cache.incr_history_sql() - if after_ts is None: - rows = await db.fetch( - """ - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 - ORDER BY created_at, id - LIMIT $2 - """, - chat_id, - limit + 1, - ) - else: - rows = await db.fetch( - """ - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 AND (created_at, id) > ($2, $3) - ORDER BY created_at, id - LIMIT $4 - """, - chat_id, - after_ts, - after_id, - limit + 1, - ) - page = rows[:limit] - items = [message_json(r) for r in page] - next_cursor = None - if len(rows) > limit: - last = page[-1] - next_cursor = encode_cursor( - {"k": "msg", "c": str(chat_id), "t": fmt_dt(last["created_at"]), "i": str(last["id"])} - ) - return {"items": items, "next_cursor": next_cursor} - - -@api.post("/chats/{chat_id}/messages") -async def create_message(request: Request, chat_id: UUID, body: CreateMessage) -> JSONResponse: - if _blank(body.text): - raise AppError(400, "invalid_request", "Некорректные поля") - user = await current_user(request) - await load_chat_for_member(chat_id, user["id"]) - message_id = uuid4() - created = datetime.now(timezone.utc) - try: - await db.execute( - """ - INSERT INTO messages (id, chat_id, sender_id, text, created_at) - VALUES ($1, $2, $3, $4, $5) - """, - message_id, - chat_id, - user["id"], - body.text, - created, - ) - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - await cache.invalidate_history(str(chat_id)) - return JSONResponse( - message_json( - { - "id": message_id, - "chat_id": chat_id, - "sender_id": user["id"], - "text": body.text, - "created_at": created, - } - ), - status_code=201, - ) - - -@api.get("/chats/{chat_id}/messages") -async def list_messages(request: Request, chat_id: UUID) -> JSONResponse: - user = await current_user(request) - await load_chat_for_member(chat_id, user["id"]) - limit = _limit(request.query_params.get("limit")) - cursor = request.query_params.get("cursor") or "" - after_ts = after_id = None - if cursor: - data = decode_cursor(cursor, "msg", str(chat_id)) - after_ts = datetime.fromisoformat(data["t"].replace("Z", "+00:00")) - after_id = UUID(data["i"]) - - async def loader(): - return await _load_messages(chat_id, after_ts, after_id, limit) - - page = await cache.get_history_page(str(chat_id), cursor, limit, loader) - return JSONResponse(page) - - -class LabFlags(BaseModel): - model_config = ConfigDict(extra="forbid") - singleflight: bool | None = None - invalidate: bool | None = None - reset_sql: bool | None = None - - -@api.get("/_lab/metrics") -async def lab_metrics() -> JSONResponse: - if not settings.lab_mode: - raise AppError(404, "not_found", "Объект не найден") - return JSONResponse( - { - "singleflight": await cache.singleflight_enabled(), - "invalidate": await cache.invalidate_enabled(), - "history_sql": await cache.history_sql_count(), - } - ) - - -@api.post("/_lab/flags") -async def lab_flags(body: LabFlags) -> JSONResponse: - if not settings.lab_mode: - raise AppError(404, "not_found", "Объект не найден") - if body.reset_sql: - await cache.reset_history_sql_count() - flags = await cache.set_flags(singleflight=body.singleflight, invalidate=body.invalidate) - return JSONResponse(flags) diff --git a/src/app/serialize.py b/src/app/serialize.py deleted file mode 100644 index 37f03710..00000000 --- a/src/app/serialize.py +++ /dev/null @@ -1,31 +0,0 @@ -from uuid import UUID - -from .timeutil import fmt_dt - - -def user_json(row) -> dict: - return { - "id": str(row["id"]), - "username": row["username"], - "display_name": row["display_name"], - "created_at": fmt_dt(row["created_at"]), - } - - -def chat_json(row, member_ids: list[UUID]) -> dict: - return { - "id": str(row["id"]), - "title": row["title"], - "member_ids": [str(x) for x in member_ids], - "created_at": fmt_dt(row["created_at"]), - } - - -def message_json(row) -> dict: - return { - "id": str(row["id"]), - "chat_id": str(row["chat_id"]), - "sender_id": str(row["sender_id"]), - "text": row["text"], - "created_at": fmt_dt(row["created_at"]), - } diff --git a/src/app/sessions.py b/src/app/sessions.py deleted file mode 100644 index bb19b760..00000000 --- a/src/app/sessions.py +++ /dev/null @@ -1,56 +0,0 @@ -import hashlib -import json -import secrets -from datetime import timedelta -from uuid import UUID - -from . import rediscli -from .config import settings -from .errors import AppError, BEARER_CHALLENGE -from .timeutil import fmt_dt, now_utc - - -def new_token() -> str: - return secrets.token_urlsafe(32) - - -def _key(token: str) -> str: - digest = hashlib.sha256(token.encode("utf-8")).hexdigest() - return "sess:" + digest - - -async def create(user_id: UUID) -> tuple[str, str]: - token = new_token() - expires = now_utc() + timedelta(seconds=settings.session_ttl_seconds) - payload = json.dumps({"user_id": str(user_id), "expires_at": fmt_dt(expires)}) - try: - await rediscli.require().set(_key(token), payload, ex=settings.session_ttl_seconds) - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - return token, fmt_dt(expires) - - -async def resolve(token: str) -> UUID: - try: - raw = await rediscli.require().get(_key(token)) - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - if not raw: - raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) - data = json.loads(raw) - return UUID(data["user_id"]) - - -async def revoke(token: str) -> None: - try: - deleted = await rediscli.require().delete(_key(token)) - except AppError: - raise - except Exception: - raise AppError(503, "unavailable", "Сервис временно недоступен") from None - if not deleted: - raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) diff --git a/src/app/timeutil.py b/src/app/timeutil.py deleted file mode 100644 index ff98eb6d..00000000 --- a/src/app/timeutil.py +++ /dev/null @@ -1,12 +0,0 @@ -from datetime import datetime, timezone - - -def now_utc() -> datetime: - return datetime.now(timezone.utc) - - -def fmt_dt(value: datetime) -> str: - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - value = value.astimezone(timezone.utc) - return value.isoformat(timespec="microseconds").replace("+00:00", "Z") diff --git a/src/app/tools/__init__.py b/src/app/tools/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/app/tools/explain.py b/src/app/tools/explain.py deleted file mode 100644 index 5178d4d1..00000000 --- a/src/app/tools/explain.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for history pagination.""" -import asyncio -import os -from uuid import uuid4 - -import asyncpg - -from app.passwords import hash_password - - -async def main() -> None: - url = os.environ["DATABASE_URL"] - conn = await asyncpg.connect(dsn=url, timeout=10, command_timeout=120) - try: - alice = uuid4() - bob = uuid4() - chat = uuid4() - await conn.execute( - "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)", - alice, - "explain_alice_" + alice.hex[:8], - "Alice", - hash_password("Example-only-not-a-real-secret-123"), - bob, - "explain_bob_" + bob.hex[:8], - "Bob", - ) - await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat") - await conn.execute( - "INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)", - chat, - alice, - bob, - ) - n = int(os.environ.get("EXPLAIN_MESSAGES", "10000")) - await conn.execute( - """ - INSERT INTO messages (id, chat_id, sender_id, text, created_at) - SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval - FROM generate_series(1, $3) AS g - """, - chat, - alice, - n, - ) - count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat) - print("seeded_messages", count, "chat_id", chat) - - query = """ - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 AND (created_at, id) > ($2, $3) - ORDER BY created_at, id - LIMIT 50 - """ - first = await conn.fetchrow( - "SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1", - chat, - ) - print("\n=== WITH index messages_chat_created_id_idx ===") - plan = await conn.fetch( - "EXPLAIN (ANALYZE, BUFFERS) " + query, - chat, - first["created_at"], - first["id"], - ) - print("\n".join(row[0] for row in plan)) - - await conn.execute("SET enable_indexscan = off") - await conn.execute("SET enable_bitmapscan = off") - await conn.execute("SET enable_indexonlyscan = off") - print("\n=== WITHOUT index scans (seq scan baseline) ===") - plan2 = await conn.fetch( - "EXPLAIN (ANALYZE, BUFFERS) " + query, - chat, - first["created_at"], - first["id"], - ) - print("\n".join(row[0] for row in plan2)) - finally: - await conn.close() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/student-tests/expiry.py b/student-tests/expiry.py deleted file mode 100644 index b242225c..00000000 --- a/student-tests/expiry.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "student-tests")) -os.environ["RUN_EXPIRY"] = "1" -from run import test_expiry # noqa: E402 - -sys.path.insert(0, str(ROOT / "tests")) -from client import Client, run # noqa: E402 - - -def main(): - c = Client(os.environ.get("BASE_URL", "http://localhost:8080")) - test_expiry(c) - print("PASS: session expiry") - - -if __name__ == "__main__": - run(main) diff --git a/student-tests/infra.py b/student-tests/infra.py deleted file mode 100644 index 6a677d45..00000000 --- a/student-tests/infra.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -"""Dependency failure and Redis-only restart checks (Docker Compose).""" -import json -import os -import subprocess -import sys -import time -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "tests")) -from client import Client, Failure, check, run # noqa: E402 - - -def compose(*args): - env_file = ROOT / ".env" - cmd = ["docker", "compose"] - if env_file.is_file(): - cmd.extend(["--env-file", str(env_file)]) - cmd.extend(args) - proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) - if proc.returncode != 0: - raise Failure("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:])) - return proc - - -def wait_ready(c: Client, seconds=90): - deadline = time.monotonic() + seconds - last = None - while time.monotonic() < deadline: - try: - c.call("GET", "/health/ready", schema="Health") - return - except Failure as exc: - last = exc - time.sleep(0.5) - raise Failure("ready not restored: %s" % last) - - -def wait_status(c: Client, path, expected, seconds=30, actor=None): - deadline = time.monotonic() + seconds - last = None - while time.monotonic() < deadline: - try: - c.call("GET", path, expected, actor=actor, schema="Health" if expected == 200 and path.startswith("/health") else None) - return - except Failure as exc: - last = exc - time.sleep(0.4) - raise Failure("did not observe %s for %s: %s" % (expected, path, last)) - - -def main(): - base = os.environ.get("BASE_URL", "http://localhost:8080") - c = Client(base, timeout=3) - alice = c.user("infra") - compose("ps") - - compose("stop", "redis") - wait_status(c, "/health/live", 200) - wait_status(c, "/health/ready", 503) - wait_status(c, "/api/v1/users/me", 503, actor=alice) - compose("start", "redis") - wait_ready(c) - c.call("GET", "/api/v1/users/me", actor=alice, schema="User") - - compose("stop", "postgres") - wait_status(c, "/health/live", 200) - wait_status(c, "/health/ready", 503) - compose("start", "postgres") - wait_ready(c) - c.call("GET", "/api/v1/users/me", actor=alice, schema="User") - - alice2, bob, _eve, chat = c.bootstrap() - msg = c.message(alice2, chat, "before-redis-restart")["json"] - compose("restart", "redis") - wait_ready(c) - me = c.call("GET", "/api/v1/users/me", actor=alice2, schema="User")["json"] - check(me["id"] == alice2["id"], "session lost after Redis restart despite AOF") - page = c.call( - "GET", - "/api/v1/chats/" + chat["id"] + "/messages?limit=100", - actor=bob, - schema="MessagePage", - )["json"] - check(any(m["id"] == msg["id"] for m in page["items"]), "history not rebuilt after cache restart") - print(json.dumps({"redis_restart": "session+history ok", "db_or_redis_down": "503"}, ensure_ascii=False)) - print("PASS: infra unavailability and redis restart") - - -if __name__ == "__main__": - run(main) diff --git a/student-tests/run.py b/student-tests/run.py deleted file mode 100644 index 9117863d..00000000 --- a/student-tests/run.py +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env python3 -"""Student checks for grades 4 and 5 (track 1: cache consistency).""" -import base64 -import json -import os -import sys -import threading -import time -import uuid -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "tests")) -from client import Client, Failure, check, run # noqa: E402 - - -def _basic(username, password): - return "Basic " + base64.b64encode((username + ":" + password).encode()).decode() - - -def lab_metrics(c: Client): - return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode()) - - -def lab_flags(c: Client, body): - return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode()) - - -def test_wrong_password_and_x_user_id(c: Client): - actor = c.user("neg") - bad = _basic(actor["registration"]["username"], "wrong-password-xx") - c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad}) - c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": actor["id"]}) - c.call("GET", "/api/v1/users/me", 401) - - -def test_logout(c: Client): - actor = c.user("out") - c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor) - c.call("GET", "/api/v1/users/me", 401, actor=actor) - - -def test_register_race(c: Client): - username = "r_" + uuid.uuid4().hex[:12] - body = { - "username": username, - "display_name": "Race", - "password": "Example-only-not-a-real-secret-123", - } - results = [] - - def once(): - local = Client(c.base) - try: - results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]) - except Failure as exc: - results.append(str(exc)) - - threads = [threading.Thread(target=once) for _ in range(16)] - for t in threads: - t.start() - for t in threads: - t.join() - created = results.count(201) - conflicted = results.count(409) - check(created == 1, "username race: expected exactly one 201, got %s" % results) - check(conflicted == 15, "username race: expected fifteen 409, got %s" % results) - - -def test_chat_rollback(c: Client): - alice = c.user("rb") - before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] - missing = str(uuid.uuid4()) - c.call( - "POST", - "/api/v1/chats", - 400, - actor=alice, - body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]}, - ) - after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] - check(len(after) == len(before), "composite chat create leaked a row after rollback") - check(all(x["title"] != "rollback-should-not-exist" for x in after), "rolled-back title became visible") - - -def test_read_after_write(c: Client): - alice, bob, _eve, chat = c.bootstrap() - path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" - lab_flags(c, {"invalidate": True, "singleflight": True, "reset_sql": True}) - first = c.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"] - page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] - check(any(m["id"] == first["id"] for m in page["items"]), "read-after-write lost the new message") - - -def test_stale_without_invalidation_then_fixed(c: Client): - alice, bob, _eve, chat = c.bootstrap() - path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50" - c.call("GET", path, actor=alice, schema="MessagePage") - lab_flags(c, {"invalidate": False, "singleflight": True}) - stale_text = "stale-probe-" + uuid.uuid4().hex - c.message(alice, chat, stale_text) - page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] - stale = all(m["text"] != stale_text for m in page["items"]) - check(stale, "expected a stale cache read when invalidation is disabled") - lab_flags(c, {"invalidate": True}) - fresh_text = "fresh-probe-" + uuid.uuid4().hex - c.message(alice, chat, fresh_text) - page2 = c.call("GET", path, actor=bob, schema="MessagePage")["json"] - check(any(m["text"] == fresh_text for m in page2["items"]), "invalidation did not restore read-after-write") - - -def test_stampede(c: Client): - alice, _bob, _eve, chat = c.bootstrap() - c.message(alice, chat, "stampede-seed") - path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50" - - def burst(n=24): - barrier = threading.Barrier(n) - - def one(_i): - local = Client(c.base) - barrier.wait() - return local.call("GET", path, actor=alice, schema="MessagePage") - - with ThreadPoolExecutor(max_workers=n) as pool: - list(pool.map(one, range(n))) - - lab_flags(c, {"singleflight": False, "invalidate": True, "reset_sql": True}) - c.message(alice, chat, "miss-" + uuid.uuid4().hex) - lab_flags(c, {"reset_sql": True, "singleflight": False}) - burst() - unprotected = lab_metrics(c)["history_sql"] - - lab_flags(c, {"singleflight": True, "reset_sql": True}) - c.message(alice, chat, "miss2-" + uuid.uuid4().hex) - lab_flags(c, {"reset_sql": True, "singleflight": True}) - burst() - protected = lab_metrics(c)["history_sql"] - print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected}, ensure_ascii=False)) - check(unprotected >= 8, "unprotected stampede did not generate enough SQL") - check(protected < unprotected, "singleflight did not reduce SQL during stampede") - check(protected <= 4, "singleflight still issued too many history SQL queries") - - -def test_expiry(c: Client): - ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) - check(ttl <= 3, "expiry test requires SESSION_TTL_SECONDS<=3 (recreate api with short TTL)") - actor = c.user("ttl") - c.call("GET", "/api/v1/users/me", actor=actor, schema="User") - time.sleep(ttl + 1.5) - c.call("GET", "/api/v1/users/me", 401, actor=actor) - - -def main(): - base = os.environ.get("BASE_URL", "http://localhost:8080") - c = Client(base) - c.call("GET", "/health/live", schema="Health") - c.call("GET", "/health/ready", schema="Health") - tests = [ - test_wrong_password_and_x_user_id, - test_logout, - test_register_race, - test_chat_rollback, - test_read_after_write, - test_stale_without_invalidation_then_fixed, - test_stampede, - ] - if os.environ.get("RUN_EXPIRY") == "1": - tests.append(test_expiry) - try: - for fn in tests: - fn(c) - print("PASS", fn.__name__) - finally: - try: - lab_flags(c, {"invalidate": True, "singleflight": True}) - except Exception: - pass - print("PASS: student grade 4/5 HTTP checks") - - -if __name__ == "__main__": - run(main) -- 2.54.0 From 8f1d7e27c7cb4a4e17e9f91003487517c74a8a96 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 7 Nov 2026 11:56:38 +0300 Subject: [PATCH 3/3] reworked --- .env.example | 10 +- .github/pull_request_template.md | 28 +-- .github/workflows/contracts.yml | 27 ++- Dockerfile | 29 +++ Makefile | 20 +- REPORT.md | 119 ++++++++++-- app/__init__.py | 1 + app/cache.py | 151 +++++++++++++++ app/config.py | 35 ++++ app/cursor_codec.py | 112 +++++++++++ app/db.py | 79 ++++++++ app/docs.py | 52 ++++++ app/errors.py | 49 +++++ app/main.py | 129 +++++++++++++ app/middleware.py | 41 +++++ app/passwords.py | 23 +++ app/rediscli.py | 40 ++++ app/routes.py | 273 +++++++++++++++++++++++++++ app/sessions.py | 56 ++++++ app/store.py | 276 ++++++++++++++++++++++++++++ app/tools/__init__.py | 1 + app/tools/explain.py | 73 ++++++++ app/validation.py | 95 ++++++++++ compose.yaml | 80 ++++++++ evidence/compose_after_restart.txt | 3 + evidence/compose_before_restart.txt | 3 + evidence/explain-history.txt | 26 +++ evidence/load.json | 17 ++ evidence/stampede.json | 10 + migrations/001_init.sql | 49 +++++ pytest.ini | 6 + requirements-test.txt | 2 + requirements.txt | 5 + scripts/restart.bat | 3 + scripts/restart.py | 20 ++ scripts/restart.sh | 9 + scripts/run_expiry.py | 55 ++++++ solve.md | 196 ++++++++++++++++++++ student_tests/__init__.py | 0 student_tests/conftest.py | 24 +++ student_tests/test_api.py | 195 ++++++++++++++++++++ student_tests/test_concurrent.py | 48 +++++ student_tests/test_expiry.py | 17 ++ student_tests/test_grade4.py | 44 +++++ student_tests/test_infra.py | 89 +++++++++ student_tests/test_track1.py | 83 +++++++++ 46 files changed, 2668 insertions(+), 35 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/cache.py create mode 100644 app/config.py create mode 100644 app/cursor_codec.py create mode 100644 app/db.py create mode 100644 app/docs.py create mode 100644 app/errors.py create mode 100644 app/main.py create mode 100644 app/middleware.py create mode 100644 app/passwords.py create mode 100644 app/rediscli.py create mode 100644 app/routes.py create mode 100644 app/sessions.py create mode 100644 app/store.py create mode 100644 app/tools/__init__.py create mode 100644 app/tools/explain.py create mode 100644 app/validation.py create mode 100644 compose.yaml create mode 100644 evidence/compose_after_restart.txt create mode 100644 evidence/compose_before_restart.txt create mode 100644 evidence/explain-history.txt create mode 100644 evidence/load.json create mode 100644 evidence/stampede.json create mode 100644 migrations/001_init.sql create mode 100644 pytest.ini create mode 100644 requirements-test.txt create mode 100644 requirements.txt create mode 100644 scripts/restart.bat create mode 100644 scripts/restart.py create mode 100644 scripts/restart.sh create mode 100644 scripts/run_expiry.py create mode 100644 solve.md create mode 100644 student_tests/__init__.py create mode 100644 student_tests/conftest.py create mode 100644 student_tests/test_api.py create mode 100644 student_tests/test_concurrent.py create mode 100644 student_tests/test_expiry.py create mode 100644 student_tests/test_grade4.py create mode 100644 student_tests/test_infra.py create mode 100644 student_tests/test_track1.py diff --git a/.env.example b/.env.example index 714c8076..b5270b86 100644 --- a/.env.example +++ b/.env.example @@ -3,9 +3,17 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +LOG_LEVEL=info +SHUTDOWN_GRACE_SECONDS=10 SESSION_TTL_SECONDS=3600 +CACHE_TTL_SECONDS=60 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url + +# Compose interpolates these. Synthetic local values only; replace after copy. +POSTGRES_PASSWORD=lab2-dev-only-change-me +DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8ceefa10..c6de4713 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 2 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Bulatsev / БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 1 — согласованность кэша -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: Starlette API из lab1; PostgreSQL как источник истины; Argon2id; Basic→opaque Bearer; Redis AOF для сессий и кэш истории с singleflight/версией. +Команды запуска из чистого clone: `cp .env.example .env && docker compose --env-file .env up -d --build` (см. REPORT.md). +Результаты проверок и ссылка на REPORT.md: smoke PASS; persistence PASS; 15 student tests PASS; infra 2 PASS; expiry PASS; EXPLAIN на 10k сообщений; stampede 24→1 SQL. Подробности в REPORT.md. +Ограничения / известные ошибки: один host; singleflight в рамках одного процесса; учебный Argon2 memory_cost. +Источники и использование ИИ: контракт курса, документация PostgreSQL/Redis; ИИ — помощь в переносе store и черновике отчёта. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289f..5236da6e 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,5 +12,28 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + application: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-test.txt + - run: cp .env.example .env + - run: docker compose --env-file .env up -d --build + - name: Wait until API is ready + run: | + for i in $(seq 1 60); do + if curl -fsS http://127.0.0.1:8080/health/ready; then exit 0; fi + sleep 2 + done + docker compose --env-file .env logs + exit 1 + - run: make test PYTHON=python + - run: chmod +x scripts/restart.sh + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python + - run: python -m pytest student_tests -q -m "not infra and not expiry" + - run: RUN_INFRA=1 python -m pytest student_tests/test_infra.py -q -m infra + - run: python scripts/run_expiry.py + - run: docker compose --env-file .env exec -T api python -m app.tools.explain diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..65e21fb0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12.8-slim-bookworm AS runtime + +WORKDIR /app + +RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app \ + && apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations +COPY contracts/openapi.json ./contracts/openapi.json + +USER app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + PYTHONUNBUFFERED=1 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + +CMD ["python", "-m", "app.main"] diff --git a/Makefile b/Makefile index 9ecb223e..3f2a813e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence test-student test-infra test-expiry explain help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -16,11 +16,23 @@ validate: $(PYTHON) scripts/validate_openapi.py test: - $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) -X utf8 tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) load: - $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) -X utf8 tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) - $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) -X utf8 tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-student: + $(PYTHON) -m pytest student_tests -q -m "not infra and not expiry" + +test-infra: + RUN_INFRA=1 $(PYTHON) -m pytest student_tests/test_infra.py -q -m infra + +test-expiry: + $(PYTHON) scripts/run_expiry.py + +explain: + docker compose --env-file .env exec -T api python -m app.tools.explain diff --git a/REPORT.md b/REPORT.md index e42f2b05..3e9e6107 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,128 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: **Bulatsev**, **БПИ2301** (ветка `solution/bpi2301-Bulatsev`) +- Целевая оценка; выбранный трек 5: **5**, трек **1 — согласованность кэша** +- Commit/PR: после push ветки `solution/bpi2301-Bulatsev` +- Источники: контракт lab2, PostgreSQL EXPLAIN, Redis AOF; код API перенесён из lab1 (Starlette/Uvicorn, валидация, cursor, graceful shutdown). Помощь ИИ — структура store/кэша и формулировки отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Prerequisites:** Docker 24+, Docker Compose v2, Python 3.9+ (локальные проверки), Git. + +**Образы:** `python:3.12.8-slim-bookworm` (API), `postgres:16.6-alpine`, `redis:7.4.2-alpine`. Лимиты compose: API 1 CPU / 512 MiB, PostgreSQL 1 CPU / 512 MiB, Redis 0.5 CPU / 128 MiB. + +```sh +git clone +cd mtusi2026lab2 +git checkout solution/bpi2301-Bulatsev +cp .env.example .env +# при необходимости замените POSTGRES_PASSWORD в .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +``` + +Готовность: `GET http://127.0.0.1:8080/health/ready` → `200 {"status":"ok"}`. Документация: `/docs`. + +**Проверки (другой терминал, Windows: `PYTHON=python` и `-X utf8` уже в Makefile):** + +```sh +make check +make test BASE_URL=http://127.0.0.1:8080 PYTHON=python +python -m pip install -r requirements-test.txt +make test-student PYTHON=python +make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python +# Windows без Git Bash: +python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://127.0.0.1:8080 +RUN_INFRA=1 python -m pytest student_tests/test_infra.py -q +python scripts/run_expiry.py +docker compose --env-file .env exec -T api python -m app.tools.explain +``` + +Миграции применяются при старте API (`schema_migrations`). Повторный старт не дублирует таблицы/данные. + +**Остановка:** `docker compose --env-file .env stop` — volumes сохраняются. +**Удаление учебных данных:** `docker compose --env-file .env down -v` (уничтожит `pg_data` и `redis_data`). + +Секреты только в локальном `.env` (gitignored). В Git нет tokens и паролей стенда, кроме синтетического примера в `.env.example`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +Клиент ──HTTP JSON──► Starlette (lab1) ──asyncpg pool──► PostgreSQL 16 + │ + └── redis.asyncio ──► Redis 7 (AOF + RDB) + сессии sess: TTL + кэш hist:::: +``` + +**Перенесено из lab1:** пакет `app/` на Starlette, middleware `X-Request-Id`, валидация JSON/Unicode, opaque cursor, Dockerfile non-root `app`, SIGTERM/`shutting_down` → 503, healthcheck. + +**Изменено в lab2:** + +| Lab1 | Lab2 | +| --- | --- | +| `InMemoryStore` | PostgreSQL: `users`, `chats`, `chat_members`, `messages` | +| `X-User-Id` | `Authorization: Bearer` (opaque token ≥256 бит) | +| регистрация без пароля | `password` + Argon2id | +| нет сессий | `POST /api/v1/auth/sessions` (Basic), `DELETE .../current` | +| состояние умирает с процессом | named volumes + Redis AOF | + +**Источник истины:** PostgreSQL. Сессии — Redis с AOF (`appendonly yes`, `appendfsync everysec`) и `save 60 1`; TTL ключа = `SESSION_TTL_SECONDS` (по умолчанию 3600, в smoke ≥300). Рестарт Redis не продлевает TTL. Кэш истории — производный: версия `histver:` увеличивается после INSERT сообщения. + +**Транзакции:** создание чата — один `transaction`: проверка участников + INSERT chat + INSERT members. Неизвестный UUID → 400, чат не появляется в списке. + +**Пул и таймауты:** `asyncpg` min 1 / max 10, `command_timeout=5`, `timeout=5`; Redis socket timeout 2 с. Запросы параметризованные (`$1…`). + +**Соответствие `.env.example`:** `DATABASE_URL` / `DATABASE_URL_FILE` → DSN Postgres; `REDIS_URL` / `CACHE_URL_FILE` по смыслу — Redis; `SESSION_STORE=redis`; `SESSION_TTL_SECONDS` читается приложением. + +**Готовность:** `/health/live` — только процесс; `/health/ready` — ping Postgres **и** Redis. Недоступность Redis на Bearer → **503**, не анонимный доступ. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` | `PASS: lab2 public API scenario` | stdout | +| Данные и сессия после рестарта | `python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.bat --base-url http://127.0.0.1:8080` | `PASS: persistence preserved confirmed data and existing session` | `evidence/compose_before_restart.txt`, `evidence/compose_after_restart.txt` (hook: `docker compose restart postgres redis api`, volumes не удалялись) | +| Миграции с нуля / повторно | повторный `up` API | `schema_migrations` пропускает `001_init.sql` | `migrations/001_init.sql`, логи старта | +| Basic только на login; Bearer; X-User-Id игнорируется | smoke + `pytest student_tests/test_api.py` | 401 на Basic/me и на `X-User-Id`; logout 204 затем 401 | `student_tests/test_api.py` | +| Password KDF | регистрация | Argon2id (`argon2-cffi`) | `app/passwords.py` | +| Гонка username | `test_register_race` | ровно один 201 и пятнадцать 409 | `student_tests/test_grade4.py` | +| Rollback чата | `test_chat_rollback` | 400, лишний чат не виден | `student_tests/test_grade4.py` | +| Индекс истории ≥10k | `docker compose exec api python -m app.tools.explain` | Index Scan, 0.061 ms vs Seq Scan 2.081 ms на 10000 сообщений | `evidence/explain-history.txt` | +| Expiry при TTL=2 | `python scripts/run_expiry.py` | 1 passed (me → 401 после TTL+1.5 с) | stdout | +| Redis/БД down | `RUN_INFRA=1 pytest student_tests/test_infra.py` | live 200, ready 503; me при Redis down 503 | 2 passed | +| Трек 1 stampede | `test_stampede_singleflight_reduces_sql` | SQL history: **24 без** singleflight, **1 с** (24 waiter, delay 250 ms) | `evidence/stampede.json` | +| Трек 1 read-after-write | `test_read_after_write` | новое сообщение сразу в GET | pytest | +| Трек 1 stale без invalidate | `test_stale_without_invalidation_then_fixed` | GET без новой строки, после включения invalidate — строка есть | pytest | +| Трек 1 рестарт кэша | `test_redis_restart_keeps_session_and_history` | тот же Bearer и то же сообщение после `compose restart redis` | pytest infra | +| Нагрузка (закрытый цикл) | `python -X utf8 tests/load.py --duration 15 --concurrency 4 --output evidence/load.json` | 337.3 successful RPS, ошибки 0, p50 9.7 / p95 27.4 / p99 33.3 ms | `evidence/load.json` | +| `make check` | `python scripts/check_contract.py` | `PASS: offline skeleton structure` | stdout | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Собственные тесты (без infra/expiry): **15 passed** за 4.37 с. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза трека 1.** Параллельные GET одной страницы истории после cache miss без координации порождают N одинаковых SQL; in-process singleflight оставляет один load. Инвалидация версией ключа обеспечивает read-after-write; без неё кэш отдаёт stale. После рестарта Redis сессия живёт за счёт AOF, страницы кэша восстанавливаются из PostgreSQL. + +**Методика.** 24 потока, `threading.Barrier`, искусственная задержка load 250 ms (только `APP_ENV=lab` через `POST /api/v1/_lab/flags`), счётчик `metrics:sql:messages_select`. Сравнение `cfg:singleflight=0/1`. Отдельно: выключение `cfg:invalidate`, POST, GET; затем включение и повтор. Затем `docker compose restart redis`. + +**Результаты.** 24 SQL → 1 SQL. Stale подтверждён при выключенной инвалидации. После рестарта Redis прежний token читает `/users/me` и историю. + +**Hardware:** Windows 10, Docker Desktop, один host. Измерения — учебные, closed-loop `tests/load.py` страдает coordinated omission. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `compose restart postgres redis api` | тот же token, то же сообщение | persistence PASS | секунды, volumes целы | +| `compose stop redis` | ready 503, Bearer не становится анонимом | 503 на `/users/me` | после `start redis` — ready | +| `compose stop postgres` | ready 503 | 503 | после `start` — ready | +| `compose restart redis` | сессия из AOF, история из PG | token и message на месте | готовность API ≤ десятка секунд | +| TTL=2 с | 401 после истечения | pytest expiry PASS | нет потери бизнес-данных | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host: отказ машины теряет всё. Redis AOF `everysec` допускает потерю <1 с сессий при crash. +- Singleflight — внутри одного процесса API (lab2: одна реплика). С lab3 нужен распределённый lock. +- Задержка 250 ms в эксперименте stampede нужна, чтобы 24 HTTP-клиента успели попасть в окно miss; без неё на localhost первый GET заполняет кэш раньше остальных. +- Argon2id с `memory_cost=8192` — учебный компромисс для гонки регистрации, не production-параметры lab7. +- Не выдерживается: потеря volume, повреждение AOF, сеть между API и Postgres. +- Улучшение stampede (24→1 SQL) **подтверждено** измерением. Линейная масштабируемость кэша на нескольких репликах — пока гипотеза. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..eeb0eb82 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger API (lab 2): PostgreSQL source of truth, Redis sessions/cache.""" diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..ed9261df --- /dev/null +++ b/app/cache.py @@ -0,0 +1,151 @@ +import asyncio +import json +from collections.abc import Awaitable, Callable +from typing import TypeVar + +from app import config, rediscli +from app.errors import AppError + +T = TypeVar("T") +CFG_SINGLEFLIGHT = "cfg:singleflight" +CFG_INVALIDATE = "cfg:invalidate" +CFG_DELAY_MS = "cfg:delay_ms" +SQL_HISTORY = "metrics:sql:messages_select" + + +class SingleFlight: + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._inflight: dict[str, asyncio.Task] = {} + + async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T: + async with self._lock: + task = self._inflight.get(key) + if task is None: + task = asyncio.create_task(factory()) + self._inflight[key] = task + owner = True + else: + owner = False + try: + return await asyncio.shield(task) + finally: + if owner: + async with self._lock: + self._inflight.pop(key, None) + + +_sf = SingleFlight() + + +async def _flag(name: str, default: bool = True) -> bool: + if not config.lab_mode(): + return default + try: + raw = await rediscli.require().get(name) + except Exception: + return default + if raw is None: + return default + return raw not in {"0", "false", "off"} + + +async def singleflight_enabled() -> bool: + return await _flag(CFG_SINGLEFLIGHT, True) + + +async def invalidate_enabled() -> bool: + return await _flag(CFG_INVALIDATE, True) + + +async def incr_history_sql() -> None: + try: + await rediscli.require().incr(SQL_HISTORY) + except Exception: + pass + + +async def history_sql_count() -> int: + try: + return int(await rediscli.require().get(SQL_HISTORY) or 0) + except Exception: + return 0 + + +async def reset_history_sql_count() -> None: + try: + await rediscli.require().set(SQL_HISTORY, 0) + except Exception: + pass + + +async def load_delay_seconds() -> float: + if not config.lab_mode(): + return 0.0 + try: + raw = await rediscli.require().get(CFG_DELAY_MS) + except Exception: + return 0.0 + try: + return max(0.0, float(raw or 0) / 1000.0) + except (TypeError, ValueError): + return 0.0 + + +async def set_flags( + *, + singleflight: bool | None = None, + invalidate: bool | None = None, + delay_ms: int | None = None, +) -> dict: + r = rediscli.require() + if singleflight is not None: + await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0") + if invalidate is not None: + await r.set(CFG_INVALIDATE, "1" if invalidate else "0") + if delay_ms is not None: + await r.set(CFG_DELAY_MS, str(int(delay_ms))) + return { + "singleflight": await singleflight_enabled(), + "invalidate": await invalidate_enabled(), + "history_sql": await history_sql_count(), + "delay_ms": int((await load_delay_seconds()) * 1000), + } + + +async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict: + r = rediscli.require() + try: + version = await r.get("histver:" + chat_id) or "0" + key = f"hist:{chat_id}:{version}:{cursor}:{limit}" + cached = await r.get(key) + if cached: + return json.loads(cached) + + async def load_and_store() -> dict: + delay = await load_delay_seconds() + if delay: + await asyncio.sleep(delay) + page = await loader() + try: + await r.set(key, json.dumps(page, ensure_ascii=False), ex=config.CACHE_TTL_SECONDS) + except Exception: + pass + return page + + if await singleflight_enabled(): + return await _sf.do(key, load_and_store) + return await load_and_store() + except AppError: + raise + except Exception: + return await loader() + + +async def invalidate_history(chat_id: str) -> None: + if not await invalidate_enabled(): + return + try: + await rediscli.require().incr("histver:" + chat_id) + except Exception: + pass diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..7103d8b5 --- /dev/null +++ b/app/config.py @@ -0,0 +1,35 @@ +import os +from pathlib import Path + + +def _env(name: str, default: str = "") -> str: + value = os.environ.get(name) + return default if value is None or value == "" else value + + +def _file_or_env(name: str, default: str = "") -> str: + path = os.environ.get(name + "_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return _env(name, default) + + +HOST = _env("APP_HOST", "0.0.0.0") +PORT = int(_env("APP_PORT", "8080")) +ENV = _env("APP_ENV", "lab") +LOG_LEVEL = _env("LOG_LEVEL", "info").lower() +SHUTDOWN_GRACE_SECONDS = float(_env("SHUTDOWN_GRACE_SECONDS", "10")) +SESSION_TTL_SECONDS = int(_env("SESSION_TTL_SECONDS", "3600")) +CACHE_TTL_SECONDS = int(_env("CACHE_TTL_SECONDS", "60")) +DATABASE_URL = _file_or_env("DATABASE_URL") +REDIS_URL = _file_or_env("REDIS_URL", "redis://redis:6379/0") +SESSION_STORE = _env("SESSION_STORE", "redis") +DB_POOL_MIN = int(_env("DB_POOL_MIN", "1")) +DB_POOL_MAX = int(_env("DB_POOL_MAX", "10")) +DB_COMMAND_TIMEOUT = float(_env("DB_COMMAND_TIMEOUT", "5")) +DB_CONNECT_TIMEOUT = float(_env("DB_CONNECT_TIMEOUT", "5")) +REDIS_SOCKET_TIMEOUT = float(_env("REDIS_SOCKET_TIMEOUT", "2")) + + +def lab_mode() -> bool: + return ENV == "lab" diff --git a/app/cursor_codec.py b/app/cursor_codec.py new file mode 100644 index 00000000..40e6ce20 --- /dev/null +++ b/app/cursor_codec.py @@ -0,0 +1,112 @@ +import base64 + +import json + +from datetime import datetime + +from typing import Any + +from uuid import UUID + + + + + +def encode_cursor(chat_id: str, created_at: str, entity_id: str) -> str: + + payload = json.dumps({"c": chat_id, "t": created_at, "i": entity_id}, separators=(",", ":")) + + return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=") + + + + + +def _validate_cursor_timestamp(value: str) -> None: + + if not value.endswith("Z"): + + raise ValueError("invalid cursor timestamp") + + try: + + datetime.fromisoformat(value[:-1] + "+00:00") + + except ValueError as exc: + + raise ValueError("invalid cursor timestamp") from exc + + + + + +def _require_str_field(data: dict, key: str) -> str: + + if key not in data: + + raise ValueError("invalid cursor") + + value = data[key] + + if not isinstance(value, str): + + raise ValueError("invalid cursor") + + return value + + + + + +def decode_cursor(cursor: str, expected_chat_id: str) -> tuple[str, str]: + + pad = "=" * (-len(cursor) % 4) + + try: + + raw = base64.urlsafe_b64decode((cursor + pad).encode("ascii")) + + data: Any = json.loads(raw.decode("utf-8")) + + except (ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc: + + raise ValueError("invalid cursor") from exc + + if not isinstance(data, dict): + + raise ValueError("invalid cursor") + + try: + + chat_id = _require_str_field(data, "c") + + created_at = _require_str_field(data, "t") + + entity_id = _require_str_field(data, "i") + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + if chat_id != expected_chat_id: + + raise ValueError("cursor chat mismatch") + + try: + + UUID(entity_id) + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + try: + + _validate_cursor_timestamp(created_at) + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + return created_at, entity_id + diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..d655513e --- /dev/null +++ b/app/db.py @@ -0,0 +1,79 @@ +from pathlib import Path + +import asyncpg + +from app import config +from app.errors import AppError + +pool: asyncpg.Pool | None = None +INTEGRITY = ( + asyncpg.UniqueViolationError, + asyncpg.ForeignKeyViolationError, + asyncpg.CheckViolationError, +) + + +def _migrations_dir() -> Path: + here = Path(__file__).resolve().parent + for candidate in (here.parent / "migrations", Path("/app/migrations")): + if candidate.is_dir(): + return candidate + raise RuntimeError("migrations directory not found") + + +async def connect() -> asyncpg.Pool: + global pool + if not config.DATABASE_URL: + raise RuntimeError("DATABASE_URL is not set") + pool = await asyncpg.create_pool( + dsn=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + command_timeout=config.DB_COMMAND_TIMEOUT, + timeout=config.DB_CONNECT_TIMEOUT, + ) + return pool + + +async def close() -> None: + global pool + if pool is not None: + await pool.close() + pool = None + + +def require() -> asyncpg.Pool: + if pool is None: + raise AppError(503, "service unavailable") + return pool + + +async def ping() -> None: + try: + async with require().acquire() as conn: + await conn.execute("SELECT 1") + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + + +async def apply_migrations() -> None: + directory = _migrations_dir() + async with require().acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")} + for path in sorted(p for p in directory.glob("*.sql") if p.is_file()): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute("INSERT INTO schema_migrations(filename) VALUES($1)", path.name) diff --git a/app/docs.py b/app/docs.py new file mode 100644 index 00000000..7f68e1d9 --- /dev/null +++ b/app/docs.py @@ -0,0 +1,52 @@ +import json +from functools import lru_cache +from pathlib import Path + +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse, Response + +_SPEC_CANDIDATES = ( + Path(__file__).resolve().parents[1] / "contracts" / "openapi.json", + Path("/app/contracts/openapi.json"), +) + +_SWAGGER_HTML = """ + + + + MTUSI Messenger — Swagger (lab2) + + + +
+ + + + +""" + + +@lru_cache(maxsize=1) +def _load_spec() -> dict: + for path in _SPEC_CANDIDATES: + if path.is_file(): + return json.loads(path.read_text(encoding="utf-8")) + return {"openapi": "3.1.0", "info": {"title": "MTUSI Messenger", "version": "1.2.0"}, "paths": {}} + + +async def openapi_json(_: Request) -> Response: + return JSONResponse(_load_spec()) + + +async def swagger_ui(_: Request) -> Response: + return HTMLResponse(_SWAGGER_HTML) diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..105e8ca3 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,49 @@ +from typing import Dict, Optional + +from starlette.responses import JSONResponse + + +ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", +} + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class AppError(Exception): + def __init__(self, status: int, message: str, www_authenticate: str | None = None) -> None: + super().__init__(message) + self.status = status + self.message = message + self.www_authenticate = www_authenticate + + +def error_response( + status: int, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, +) -> JSONResponse: + code = ERROR_CODES.get(status, "internal_error") + headers = {"X-Request-Id": request_id, "Content-Type": "application/json"} + if extra_headers: + headers.update(extra_headers) + return JSONResponse( + status_code=status, + headers=headers, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + ) + + +def from_app_error(exc: AppError, request_id: str) -> JSONResponse: + extra = {"WWW-Authenticate": exc.www_authenticate} if exc.www_authenticate else None + return error_response(exc.status, exc.message, request_id, extra) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..de77f6cd --- /dev/null +++ b/app/main.py @@ -0,0 +1,129 @@ +import asyncio +import logging +import signal +import sys +import uuid + +import uvicorn +from starlette.applications import Starlette +from starlette.exceptions import HTTPException +from starlette.requests import Request +from starlette.responses import Response +from starlette.routing import Route + +from app import config, db, rediscli +from app.docs import openapi_json, swagger_ui +from app.errors import AppError, error_response, from_app_error +from app.middleware import RequestContextMiddleware +from app.routes import ( + create_chat, + create_message, + create_session, + current_user, + get_chat, + health_live, + health_ready, + lab_flags, + lab_metrics, + list_chats, + list_messages, + register_user, + revoke_session, +) + + +def _request_id(request: Request) -> str: + existing = getattr(request.state, "request_id", None) + if existing: + return existing + return request.headers.get("X-Request-Id") or str(uuid.uuid4()) + + +async def not_found_handler(request: Request, exc: HTTPException) -> Response: + return error_response(404, "not found", _request_id(request)) + + +async def app_error_handler(request: Request, exc: AppError) -> Response: + return from_app_error(exc, _request_id(request)) + + +async def startup() -> None: + await db.connect() + await db.apply_migrations() + await rediscli.connect() + + +async def shutdown() -> None: + await rediscli.close() + await db.close() + + +def build_app() -> Starlette: + app = Starlette( + routes=[ + Route("/docs", swagger_ui, methods=["GET"]), + Route("/openapi.json", openapi_json, methods=["GET"]), + Route("/health/live", health_live, methods=["GET"]), + Route("/health/ready", health_ready, methods=["GET"]), + Route("/api/v1/users", register_user, methods=["POST"]), + Route("/api/v1/users/me", current_user, methods=["GET"]), + Route("/api/v1/auth/sessions", create_session, methods=["POST"]), + Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]), + Route("/api/v1/chats", create_chat, methods=["POST"]), + Route("/api/v1/chats", list_chats, methods=["GET"]), + Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), + Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]), + Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]), + ], + exception_handlers={404: not_found_handler, AppError: app_error_handler}, + on_startup=[startup], + on_shutdown=[shutdown], + ) + app.state.shutting_down = False + app.add_middleware(RequestContextMiddleware) + return app + + +def configure_logging() -> None: + logging.basicConfig( + level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + ) + + +def main() -> None: + configure_logging() + app = build_app() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + server = uvicorn.Server( + uvicorn.Config( + app, + host=config.HOST, + port=config.PORT, + log_level=config.LOG_LEVEL, + access_log=False, + timeout_graceful_shutdown=int(config.SHUTDOWN_GRACE_SECONDS), + ) + ) + + def _begin_shutdown() -> None: + logging.getLogger("messenger").info("shutdown signal received") + app.state.shutting_down = True + server.should_exit = True + + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, _begin_shutdown) + except NotImplementedError: + signal.signal(sig, lambda *_: _begin_shutdown()) + + loop.run_until_complete(server.serve()) + + +if __name__ == "__main__": + main() diff --git a/app/middleware.py b/app/middleware.py new file mode 100644 index 00000000..76f3d06b --- /dev/null +++ b/app/middleware.py @@ -0,0 +1,41 @@ +import logging +import time +import uuid +from typing import Callable + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from app.errors import AppError, error_response, from_app_error + +logger = logging.getLogger("messenger.access") + + +class RequestContextMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next: Callable) -> Response: + request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = request_id + + if request.app.state.shutting_down and request.url.path not in ("/health/live",): + return error_response(503, "service is shutting down", request_id) + + started = time.perf_counter() + try: + response = await call_next(request) + except AppError as exc: + return from_app_error(exc, request_id) + except Exception: + logger.exception("request_id=%s unhandled error path=%s", request_id, request.url.path) + return error_response(500, "internal error", request_id) + elapsed_ms = (time.perf_counter() - started) * 1000 + response.headers["X-Request-Id"] = request_id + logger.info( + "request_id=%s method=%s path=%s status=%s duration_ms=%.2f", + request_id, + request.method, + request.url.path, + response.status_code, + elapsed_ms, + ) + return response diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..8a640cd0 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,23 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + +hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1) +_dummy = None + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def dummy_hash() -> str: + global _dummy + if _dummy is None: + _dummy = hash_password("timing-dummy-password") + return _dummy + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError, TypeError, ValueError): + return False diff --git a/app/rediscli.py b/app/rediscli.py new file mode 100644 index 00000000..de173117 --- /dev/null +++ b/app/rediscli.py @@ -0,0 +1,40 @@ +import redis.asyncio as redis + +from app import config +from app.errors import AppError + +client: redis.Redis | None = None + + +async def connect() -> redis.Redis: + global client + client = redis.from_url( + config.REDIS_URL, + socket_connect_timeout=config.REDIS_SOCKET_TIMEOUT, + socket_timeout=config.REDIS_SOCKET_TIMEOUT, + decode_responses=True, + ) + await client.ping() + return client + + +async def close() -> None: + global client + if client is not None: + await client.close() + client = None + + +def require() -> redis.Redis: + if client is None: + raise AppError(503, "service unavailable") + return client + + +async def ping() -> None: + try: + await require().ping() + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None diff --git a/app/routes.py b/app/routes.py new file mode 100644 index 00000000..a5539b72 --- /dev/null +++ b/app/routes.py @@ -0,0 +1,273 @@ +import base64 +import binascii + +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +from app import cache, config, db, rediscli, sessions, store +from app.cursor_codec import decode_cursor, encode_cursor +from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response +from app.passwords import dummy_hash, hash_password, verify_password +from app.validation import ( + parse_json_body, + require_exact_keys, + validate_display_name, + validate_limit, + validate_message_text, + validate_password, + validate_title, + validate_username, + validate_uuid, +) + + +def _json(data: dict, request_id: str, status: int = 200) -> JSONResponse: + return JSONResponse(status_code=status, headers={"X-Request-Id": request_id}, content=data) + + +def _empty(request_id: str) -> Response: + return Response(status_code=204, headers={"X-Request-Id": request_id}) + + +async def _bearer_user(request: Request) -> dict: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "authentication required", BEARER_CHALLENGE) + user_id = str(await sessions.resolve(rest.strip())) + user = await store.get_user(user_id) + if user is None: + raise AppError(401, "authentication required", BEARER_CHALLENGE) + return user + + +async def health_live(request: Request) -> Response: + return _json({"status": "ok"}, request.state.request_id) + + +async def health_ready(request: Request) -> Response: + rid = request.state.request_id + if request.app.state.shutting_down: + return error_response(503, "service is shutting down", rid) + await db.ping() + await rediscli.ping() + return _json({"status": "ok"}, rid) + + +async def register_user(request: Request) -> Response: + rid = request.state.request_id + if request.headers.get("content-type", "").split(";")[0].strip().lower() not in ("application/json", ""): + if request.headers.get("content-type"): + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}): + return error_response(400, "invalid user payload", rid) + if validate_username(body["username"]) or validate_display_name(body["display_name"]) or validate_password(body["password"]): + return error_response(400, "invalid user fields", rid) + user, conflict = await store.create_user(body["username"], body["display_name"], hash_password(body["password"])) + if conflict: + return error_response(409, "username already taken", rid) + assert user is not None + return _json(user, rid, 201) + + +async def current_user(request: Request) -> Response: + user = await _bearer_user(request) + return _json(user, request.state.request_id) + + +async def create_session(request: Request) -> Response: + rid = request.state.request_id + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "basic" or not rest.strip(): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + try: + decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeError, ValueError): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + username, sep, password = decoded.partition(":") + if not sep: + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + row = await store.get_user_by_username(username) + password_hash = row["password_hash"] if row else dummy_hash() + if row is None or not verify_password(password_hash, password): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + token, expires_at = await sessions.create(row["id"]) + return _json( + { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": store.user_dict(row), + }, + rid, + 201, + ) + + +async def revoke_session(request: Request) -> Response: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "authentication required", BEARER_CHALLENGE) + await sessions.revoke(rest.strip()) + return _empty(request.state.request_id) + + +async def create_chat(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if ct and ct != "application/json": + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"title", "member_ids"}, {"title", "member_ids"}): + return error_response(400, "invalid chat payload", rid) + if validate_title(body["title"]): + return error_response(400, "invalid title", rid) + members = body["member_ids"] + if not isinstance(members, list) or not members: + return error_response(400, "invalid member_ids", rid) + if len(members) > 100: + return error_response(400, "too many members", rid) + for mid in members: + if validate_uuid(mid): + return error_response(400, "invalid member id", rid) + chat, create_err = await store.create_chat(body["title"], members, user["id"]) + if create_err == "caller not in members": + return error_response(400, "caller must be a member", rid) + if create_err in ("duplicate members", "unknown member"): + return error_response(400, "invalid members", rid) + assert chat is not None + return _json(chat, rid, 201) + + +async def list_chats(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + limit, limit_err = validate_limit(request.query_params.get("limit")) + if limit_err: + return error_response(400, limit_err, rid) + assert limit is not None + cursor_raw = request.query_params.get("cursor") + after = None + if cursor_raw is not None: + if not cursor_raw.strip(): + return error_response(400, "invalid cursor", rid) + try: + created_at, entity_id = decode_cursor(cursor_raw, user["id"]) + except ValueError: + return error_response(400, "invalid cursor", rid) + after = (created_at, entity_id) + items, next_key = await store.list_chats(user["id"], limit, after) + next_cursor = None + if next_key is not None: + next_cursor = encode_cursor(user["id"], next_key[0], next_key[1]) + return _json({"items": items, "next_cursor": next_cursor}, rid) + + +async def get_chat(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + chat = await store.get_chat(chat_id, user["id"]) + if chat is None: + return error_response(404, "chat not found", rid) + return _json(chat, rid) + + +async def create_message(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if ct and ct != "application/json": + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"text"}, {"text"}): + return error_response(400, "invalid message payload", rid) + if validate_message_text(body["text"]): + return error_response(400, "invalid message text", rid) + message, msg_err = await store.add_message(chat_id, user["id"], body["text"]) + if msg_err == "not_found": + return error_response(404, "chat not found", rid) + assert message is not None + return _json(message, rid, 201) + + +async def list_messages(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + limit, limit_err = validate_limit(request.query_params.get("limit")) + if limit_err: + return error_response(400, limit_err, rid) + assert limit is not None + cursor_raw = request.query_params.get("cursor") + after = None + if cursor_raw is not None: + if not cursor_raw.strip(): + return error_response(400, "invalid cursor", rid) + try: + created_at, entity_id = decode_cursor(cursor_raw, chat_id) + except ValueError: + return error_response(400, "invalid cursor", rid) + after = (created_at, entity_id) + page, next_key, list_err = await store.list_messages(chat_id, user["id"], limit, after) + if list_err == "not_found": + return error_response(404, "chat not found", rid) + assert page is not None + next_cursor = None + if next_key is not None: + next_cursor = encode_cursor(chat_id, next_key[0], next_key[1]) + return _json({"items": page, "next_cursor": next_cursor}, rid) + + +async def lab_metrics(request: Request) -> Response: + if not config.lab_mode(): + return error_response(404, "not found", request.state.request_id) + return _json( + { + "singleflight": await cache.singleflight_enabled(), + "invalidate": await cache.invalidate_enabled(), + "history_sql": await cache.history_sql_count(), + }, + request.state.request_id, + ) + + +async def lab_flags(request: Request) -> Response: + if not config.lab_mode(): + return error_response(404, "not found", request.state.request_id) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, request.state.request_id) + assert body is not None + if body.get("reset_sql"): + await cache.reset_history_sql_count() + flags = await cache.set_flags( + singleflight=body.get("singleflight"), + invalidate=body.get("invalidate"), + delay_ms=body.get("delay_ms"), + ) + return _json(flags, request.state.request_id) diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..b0e30b11 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,56 @@ +import hashlib +import json +import secrets +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from app import config, rediscli +from app.errors import AppError, BEARER_CHALLENGE + + +def new_token() -> str: + return secrets.token_urlsafe(32) + + +def _key(token: str) -> str: + return "sess:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _fmt(dt: datetime) -> str: + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +async def create(user_id: UUID) -> tuple[str, str]: + token = new_token() + expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + payload = json.dumps({"user_id": str(user_id), "expires_at": _fmt(expires)}) + try: + await rediscli.require().set(_key(token), payload, ex=config.SESSION_TTL_SECONDS) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + return token, _fmt(expires) + + +async def resolve(token: str) -> UUID: + try: + raw = await rediscli.require().get(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if not raw: + raise AppError(401, "authentication required", BEARER_CHALLENGE) + return UUID(json.loads(raw)["user_id"]) + + +async def revoke(token: str) -> None: + try: + deleted = await rediscli.require().delete(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if not deleted: + raise AppError(401, "authentication required", BEARER_CHALLENGE) diff --git a/app/store.py b/app/store.py new file mode 100644 index 00000000..5c580f17 --- /dev/null +++ b/app/store.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import List, Optional, Tuple +from uuid import UUID, uuid4 + +import asyncpg + +from app import cache, db +from app.errors import AppError + + +def format_ts(dt: datetime) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def user_dict(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_ts(row["created_at"]), + } + + +def chat_dict(row, member_ids: List[str]) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": format_ts(row["created_at"]), + } + + +def message_dict(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_ts(row["created_at"]), + } + + +async def _run(coro): + try: + return await coro + except AppError: + raise + except db.INTEGRITY: + raise + except Exception: + raise AppError(503, "service unavailable") from None + + +async def create_user(username: str, display_name: str, password_hash: str) -> Tuple[Optional[dict], Optional[str]]: + user_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + await conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + user_id, + username, + display_name, + password_hash, + created, + ) + except asyncpg.UniqueViolationError: + return None, "conflict" + except Exception: + raise AppError(503, "service unavailable") from None + return user_dict({"id": user_id, "username": username, "display_name": display_name, "created_at": created}), None + + +async def get_user(user_id: str) -> Optional[dict]: + row = await _run( + _fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", UUID(user_id)) + ) + return user_dict(row) if row else None + + +async def get_user_by_username(username: str): + return await _run( + _fetchrow( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1", + username, + ) + ) + + +async def _fetchrow(sql: str, *args): + async with db.require().acquire() as conn: + return await conn.fetchrow(sql, *args) + + +async def _fetch(sql: str, *args): + async with db.require().acquire() as conn: + return await conn.fetch(sql, *args) + + +async def member_ids_of(chat_id: UUID) -> List[str]: + rows = await _run(_fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id)) + return [str(r["user_id"]) for r in rows] + + +async def create_chat(title: str, member_ids: List[str], caller_id: str) -> Tuple[Optional[dict], Optional[str]]: + if caller_id not in member_ids: + return None, "caller not in members" + if len(set(member_ids)) != len(member_ids): + return None, "duplicate members" + ids = [UUID(x) for x in member_ids] + chat_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", ids) + if len(existing) != len(ids): + return None, "unknown member" + await conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", + chat_id, + title, + created, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, mid) for mid in ids], + ) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + return chat_dict({"id": chat_id, "title": title, "created_at": created}, member_ids), None + + +async def get_chat(chat_id: str, user_id: str) -> Optional[dict]: + row = await load_chat_for_member(chat_id, user_id) + if row is None: + return None + return chat_dict(row, await member_ids_of(UUID(chat_id))) + + +async def load_chat_for_member(chat_id: str, user_id: str): + return await _run( + _fetchrow( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 + WHERE c.id = $1 + """, + UUID(chat_id), + UUID(user_id), + ) + ) + + +async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] = None): + args: list = [UUID(user_id)] + clause = "" + if after is not None: + clause = "AND (c.created_at, c.id) > ($2, $3)" + args.extend([datetime.fromisoformat(after[0].replace("Z", "+00:00")), UUID(after[1])]) + rows = await _run( + _fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1 + WHERE 1=1 {clause} + ORDER BY c.created_at, c.id + LIMIT {int(limit) + 1} + """, + *args, + ) + ) + page = rows[:limit] + items = [chat_dict(row, await member_ids_of(row["id"])) for row in page] + next_key = None + if len(rows) > limit: + last = page[-1] + next_key = (format_ts(last["created_at"]), str(last["id"])) + return items, next_key + + +async def add_message(chat_id: str, sender_id: str, text: str) -> Tuple[Optional[dict], Optional[str]]: + if await load_chat_for_member(chat_id, sender_id) is None: + return None, "not_found" + message_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + message_id, + UUID(chat_id), + UUID(sender_id), + text, + created, + ) + except Exception: + raise AppError(503, "service unavailable") from None + await cache.invalidate_history(chat_id) + return message_dict( + { + "id": message_id, + "chat_id": UUID(chat_id), + "sender_id": UUID(sender_id), + "text": text, + "created_at": created, + } + ), None + + +async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[Tuple[str, str]] = None): + if await load_chat_for_member(chat_id, user_id) is None: + return None, None, "not_found" + cursor = "" + after_ts = after_id = None + if after is not None: + cursor = after[0] + ":" + after[1] + after_ts = datetime.fromisoformat(after[0].replace("Z", "+00:00")) + after_id = UUID(after[1]) + + async def loader() -> dict: + await cache.incr_history_sql() + if after_ts is None: + rows = await _run( + _fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id + LIMIT $2 + """, + UUID(chat_id), + limit + 1, + ) + ) + else: + rows = await _run( + _fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT $4 + """, + UUID(chat_id), + after_ts, + after_id, + limit + 1, + ) + ) + page = rows[:limit] + items = [message_dict(r) for r in page] + next_key = None + if len(rows) > limit: + last = page[-1] + next_key = (format_ts(last["created_at"]), str(last["id"])) + return {"items": items, "next_key": next_key} + + page = await cache.get_history_page(chat_id, cursor, limit, loader) + return page["items"], page.get("next_key"), None diff --git a/app/tools/__init__.py b/app/tools/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/app/tools/__init__.py @@ -0,0 +1 @@ + diff --git a/app/tools/explain.py b/app/tools/explain.py new file mode 100644 index 00000000..e649ad14 --- /dev/null +++ b/app/tools/explain.py @@ -0,0 +1,73 @@ +"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for keyset pagination.""" +import asyncio +import os +from uuid import uuid4 + +import asyncpg + +from app.passwords import hash_password + + +async def main() -> None: + conn = await asyncpg.connect(dsn=os.environ["DATABASE_URL"], timeout=10, command_timeout=120) + try: + alice = uuid4() + bob = uuid4() + chat = uuid4() + pwd = hash_password("Example-only-not-a-real-secret-123") + await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)", + alice, + "explain_a_" + alice.hex[:8], + "Alice", + pwd, + bob, + "explain_b_" + bob.hex[:8], + "Bob", + ) + await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat") + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)", + chat, + alice, + bob, + ) + n = int(os.environ.get("EXPLAIN_MESSAGES", "10000")) + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval + FROM generate_series(1, $3) AS g + """, + chat, + alice, + n, + ) + count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat) + print("seeded_messages", count, "chat_id", chat) + first = await conn.fetchrow( + "SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1", + chat, + ) + query = """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT 50 + """ + print("\n=== WITH index messages_chat_created_id_idx ===") + plan = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"]) + print("\n".join(row[0] for row in plan)) + await conn.execute("SET enable_indexscan = off") + await conn.execute("SET enable_bitmapscan = off") + await conn.execute("SET enable_indexonlyscan = off") + print("\n=== WITHOUT index scans (seq scan baseline) ===") + plan2 = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"]) + print("\n".join(row[0] for row in plan2)) + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..6b4f39db --- /dev/null +++ b/app/validation.py @@ -0,0 +1,95 @@ +import json +import re +from typing import Any, Dict, Optional, Tuple +from uuid import UUID + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +def parse_json_body(raw: bytes) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + if not raw: + return None, "empty body" + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None, "malformed JSON" + if not isinstance(data, dict): + return None, "JSON body must be an object" + return data, None + + +def require_exact_keys(data: Dict[str, Any], allowed: set, required: set) -> Optional[str]: + keys = set(data.keys()) + if keys - allowed: + return "unexpected fields" + if required - keys: + return "missing required fields" + return None + + +def validate_username(value: Any) -> Optional[str]: + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + return "invalid username" + return None + + +def validate_display_name(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid display_name" + if not value.strip(): + return "display_name must not be blank" + if len(value) > 100: + return "display_name too long" + return None + + +def validate_password(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid password" + if not value.strip(): + return "password must not be blank" + if len(value) < 12 or len(value) > 128: + return "password length" + return None + + +def validate_title(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid title" + if not value.strip(): + return "title must not be blank" + if len(value) > 100: + return "title too long" + return None + + +def validate_uuid(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid uuid" + try: + UUID(value) + except ValueError: + return "invalid uuid" + return None + + +def validate_message_text(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid text" + if not value.strip(): + return "text must not be blank" + if len(value) > 4000: + return "text too long" + return None + + +def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]: + if value is None: + return default, None + try: + limit = int(value) + except ValueError: + return None, "invalid limit" + if limit < 1 or limit > 100: + return None, "limit out of range" + return limit, None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..d61e1dba --- /dev/null +++ b/compose.yaml @@ -0,0 +1,80 @@ +services: + api: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab2 + ports: + - "127.0.0.1:8080:8080" + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + LOG_LEVEL: info + SHUTDOWN_GRACE_SECONDS: "10" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60} + SESSION_STORE: redis + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + REDIS_URL: redis://redis:6379/0 + DB_POOL_MIN: "1" + DB_POOL_MAX: "10" + DB_COMMAND_TIMEOUT: "5" + DB_CONNECT_TIMEOUT: "5" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + mem_limit: 512m + cpus: "1.0" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 20s + + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: messenger + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 512m + cpus: "1.0" + + redis: + image: redis:7.4.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --save + - "60" + - "1" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 128m + cpus: "0.50" + +volumes: + pg_data: + redis_data: diff --git a/evidence/compose_after_restart.txt b/evidence/compose_after_restart.txt new file mode 100644 index 00000000..8ef15e83 --- /dev/null +++ b/evidence/compose_after_restart.txt @@ -0,0 +1,3 @@ +{"Command":"\"python -m app.main\"","CreatedAt":"2026-11-07 11:46:43 +0300 MSK","ExitCode":0,"Health":"starting","ID":"35a7c6412cc4","Image":"mtusi-messenger:lab2","Labels":"com.docker.compose.container-number=1,com.docker.compose.project.config_files=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\compose.yaml,com.docker.compose.project.environment_file=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\.env,com.docker.compose.project.working_dir=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2,com.docker.compose.replace=api-1,com.docker.compose.depends_on=redis:service_healthy:false,postgres:service_healthy:false,com.docker.compose.image=sha256:68ca44864e87cf6b2e15ff1ae2a62a58e754282d7cfd58e7c44e7e3820eb4129,com.docker.compose.oneoff=False,com.docker.compose.project=mtusi2026lab2,com.docker.compose.service=api,com.docker.compose.version=5.1.4,desktop.docker.io/ports.scheme=v2,desktop.docker.io/ports/8080/tcp=127.0.0.1:8080,com.docker.compose.config-hash=a7a455b20e0dbc0b8ebe9657814869a6dd93509709073efef2180caaa2708102","LocalVolumes":"0","Mounts":"","Name":"mtusi2026lab2-api-1","Names":"mtusi2026lab2-api-1","Networks":"mtusi2026lab2_default","Ports":"127.0.0.1:8080-\u003e8080/tcp","Project":"mtusi2026lab2","Publishers":[{"URL":"127.0.0.1","TargetPort":8080,"PublishedPort":8080,"Protocol":"tcp"}],"RunningFor":"24 seconds ago","Service":"api","Size":"0B","State":"running","Status":"Up 1 second (health: starting)"} +{"Command":"\"docker-entrypoint.sΓǪ\"","CreatedAt":"2026-11-07 11:41:21 +0300 MSK","ExitCode":0,"Health":"starting","ID":"e56b24d17f33","Image":"postgres:16.6-alpine","Labels":"com.docker.compose.config-hash=f1371900a17bcdc05dede1af7f1eb40384c58a16f0e7dd521f9bf9fd59cba4d4,com.docker.compose.container-number=1,com.docker.compose.image=sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef,com.docker.compose.oneoff=False,com.docker.compose.project=mtusi2026lab2,com.docker.compose.project.working_dir=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2,com.docker.compose.version=5.1.4,desktop.docker.io/ports.scheme=v2,com.docker.compose.depends_on=,com.docker.compose.project.config_files=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\compose.yaml,com.docker.compose.project.environment_file=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\.env,com.docker.compose.service=postgres","LocalVolumes":"1","Mounts":"mtusi2026lab2_ΓǪ","Name":"mtusi2026lab2-postgres-1","Names":"mtusi2026lab2-postgres-1","Networks":"mtusi2026lab2_default","Ports":"5432/tcp","Project":"mtusi2026lab2","Publishers":[{"URL":"","TargetPort":5432,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"5 minutes ago","Service":"postgres","Size":"0B","State":"running","Status":"Up 1 second (health: starting)"} +{"Command":"\"docker-entrypoint.sΓǪ\"","CreatedAt":"2026-11-07 11:41:21 +0300 MSK","ExitCode":0,"Health":"starting","ID":"e36d755039e4","Image":"redis:7.4.2-alpine","Labels":"com.docker.compose.config-hash=c3421b01b047dac5406b7325d9879914f4830500c975d999822051d3926fdf3c,com.docker.compose.container-number=1,com.docker.compose.depends_on=,com.docker.compose.image=sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\compose.yaml,desktop.docker.io/ports.scheme=v2,com.docker.compose.project=mtusi2026lab2,com.docker.compose.project.environment_file=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\.env,com.docker.compose.project.working_dir=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2,com.docker.compose.service=redis,com.docker.compose.version=5.1.4","LocalVolumes":"1","Mounts":"mtusi2026lab2_ΓǪ","Name":"mtusi2026lab2-redis-1","Names":"mtusi2026lab2-redis-1","Networks":"mtusi2026lab2_default","Ports":"6379/tcp","Project":"mtusi2026lab2","Publishers":[{"URL":"","TargetPort":6379,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"5 minutes ago","Service":"redis","Size":"0B","State":"running","Status":"Up 1 second (health: starting)"} diff --git a/evidence/compose_before_restart.txt b/evidence/compose_before_restart.txt new file mode 100644 index 00000000..9b965a4f --- /dev/null +++ b/evidence/compose_before_restart.txt @@ -0,0 +1,3 @@ +{"Command":"\"python -m app.main\"","CreatedAt":"2026-11-07 11:46:43 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"35a7c6412cc4","Image":"mtusi-messenger:lab2","Labels":"com.docker.compose.project.working_dir=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2,com.docker.compose.replace=api-1,com.docker.compose.service=api,desktop.docker.io/ports.scheme=v2,com.docker.compose.container-number=1,com.docker.compose.depends_on=redis:service_healthy:false,postgres:service_healthy:false,com.docker.compose.image=sha256:68ca44864e87cf6b2e15ff1ae2a62a58e754282d7cfd58e7c44e7e3820eb4129,com.docker.compose.oneoff=False,com.docker.compose.project=mtusi2026lab2,com.docker.compose.project.environment_file=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\.env,com.docker.compose.version=5.1.4,desktop.docker.io/ports/8080/tcp=127.0.0.1:8080,com.docker.compose.config-hash=a7a455b20e0dbc0b8ebe9657814869a6dd93509709073efef2180caaa2708102,com.docker.compose.project.config_files=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\compose.yaml","LocalVolumes":"0","Mounts":"","Name":"mtusi2026lab2-api-1","Names":"mtusi2026lab2-api-1","Networks":"mtusi2026lab2_default","Ports":"127.0.0.1:8080-\u003e8080/tcp","Project":"mtusi2026lab2","Publishers":[{"URL":"127.0.0.1","TargetPort":8080,"PublishedPort":8080,"Protocol":"tcp"}],"RunningFor":"22 seconds ago","Service":"api","Size":"0B","State":"running","Status":"Up 19 seconds (healthy)"} +{"Command":"\"docker-entrypoint.sΓǪ\"","CreatedAt":"2026-11-07 11:41:21 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"e56b24d17f33","Image":"postgres:16.6-alpine","Labels":"com.docker.compose.service=postgres,com.docker.compose.version=5.1.4,desktop.docker.io/ports.scheme=v2,com.docker.compose.config-hash=f1371900a17bcdc05dede1af7f1eb40384c58a16f0e7dd521f9bf9fd59cba4d4,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\compose.yaml,com.docker.compose.depends_on=,com.docker.compose.image=sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef,com.docker.compose.project=mtusi2026lab2,com.docker.compose.project.environment_file=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\.env,com.docker.compose.project.working_dir=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2","LocalVolumes":"1","Mounts":"mtusi2026lab2_ΓǪ","Name":"mtusi2026lab2-postgres-1","Names":"mtusi2026lab2-postgres-1","Networks":"mtusi2026lab2_default","Ports":"5432/tcp","Project":"mtusi2026lab2","Publishers":[{"URL":"","TargetPort":5432,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"5 minutes ago","Service":"postgres","Size":"0B","State":"running","Status":"Up 5 minutes (healthy)"} +{"Command":"\"docker-entrypoint.sΓǪ\"","CreatedAt":"2026-11-07 11:41:21 +0300 MSK","ExitCode":0,"Health":"healthy","ID":"e36d755039e4","Image":"redis:7.4.2-alpine","Labels":"com.docker.compose.config-hash=c3421b01b047dac5406b7325d9879914f4830500c975d999822051d3926fdf3c,com.docker.compose.image=sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952,com.docker.compose.project=mtusi2026lab2,com.docker.compose.project.working_dir=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2,desktop.docker.io/ports.scheme=v2,com.docker.compose.container-number=1,com.docker.compose.depends_on=,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\compose.yaml,com.docker.compose.project.environment_file=C:\\Users\\david\\Desktop\\mtuci\\highload\\lab2\\mtusi2026lab2\\.env,com.docker.compose.service=redis,com.docker.compose.version=5.1.4","LocalVolumes":"1","Mounts":"mtusi2026lab2_ΓǪ","Name":"mtusi2026lab2-redis-1","Names":"mtusi2026lab2-redis-1","Networks":"mtusi2026lab2_default","Ports":"6379/tcp","Project":"mtusi2026lab2","Publishers":[{"URL":"","TargetPort":6379,"PublishedPort":0,"Protocol":"tcp"}],"RunningFor":"5 minutes ago","Service":"redis","Size":"0B","State":"running","Status":"Up 5 minutes (healthy)"} diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 00000000..3f393507 --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,26 @@ +seeded_messages 10000 chat_id 364e2ede-c155-44e4-9a70-4f1a7cfcac5c + +=== WITH index messages_chat_created_id_idx === +Limit (cost=0.28..8.30 rows=1 width=86) (actual time=0.012..0.017 rows=50 loops=1) + Buffers: shared hit=3 + -> Index Scan using messages_chat_created_id_idx on messages (cost=0.28..8.30 rows=1 width=86) (actual time=0.011..0.014 rows=50 loops=1) + Index Cond: ((chat_id = '364e2ede-c155-44e4-9a70-4f1a7cfcac5c'::uuid) AND (ROW(created_at, id) > ROW('2026-11-07 08:47:19.224775+00'::timestamp with time zone, 'b63c0a90-e11b-4742-b7a3-122f5ec164ca'::uuid))) + Buffers: shared hit=3 +Planning: + Buffers: shared hit=36 +Planning Time: 0.307 ms +Execution Time: 0.061 ms + +=== WITHOUT index scans (seq scan baseline) === +Limit (cost=206.18..206.19 rows=1 width=86) (actual time=2.041..2.046 rows=50 loops=1) + Buffers: shared hit=130 read=11 + -> Sort (cost=206.18..206.19 rows=1 width=86) (actual time=2.040..2.042 rows=50 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 38kB + Buffers: shared hit=130 read=11 + -> Seq Scan on messages (cost=0.00..206.17 rows=1 width=86) (actual time=0.080..0.805 rows=9999 loops=1) + Filter: ((chat_id = '364e2ede-c155-44e4-9a70-4f1a7cfcac5c'::uuid) AND (ROW(created_at, id) > ROW('2026-11-07 08:47:19.224775+00'::timestamp with time zone, 'b63c0a90-e11b-4742-b7a3-122f5ec164ca'::uuid))) + Rows Removed by Filter: 350 + Buffers: shared hit=124 read=11 +Planning Time: 0.053 ms +Execution Time: 2.081 ms diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..cc1cd510 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 2, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.052, + "concurrency": 4, + "successful_requests": 5077, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 337.306, + "successful_latency_ms": { + "p50": 9.684, + "p95": 27.385, + "p99": 33.26 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/stampede.json b/evidence/stampede.json new file mode 100644 index 00000000..b2c87c4d --- /dev/null +++ b/evidence/stampede.json @@ -0,0 +1,10 @@ +{ + "hypothesis": "Without singleflight, concurrent cache misses on the same history page issue one SQL SELECT per waiter; with in-process singleflight they share one load.", + "concurrency": 24, + "artificial_miss_delay_ms": 250, + "stampede_sql_without_singleflight": 24, + "stampede_sql_with_singleflight": 1, + "read_after_write": "POST message increments histver; next GET misses and returns the new row", + "stale_when_invalidate_disabled": "GET after POST omitted the new text until invalidate was re-enabled", + "redis_restart": "AOF restored the Bearer session; history was rebuilt from PostgreSQL" +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..2501deea --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,49 @@ +-- Lab2 schema (written before application queries). +-- Source of truth: PostgreSQL for users, chats, membership, messages. +-- Transaction boundary: INSERT chat + INSERT members in one transaction (rollback together). +-- Sessions are stored in Redis with AOF, not in this database. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE UNIQUE INDEX users_username_key ON users (username); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id); +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset history: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..ce023a9a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = student_tests +pythonpath = . +markers = + infra: compose stop/restart of redis or postgres + expiry: requires SESSION_TTL_SECONDS<=3 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..f7f80320 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==8.3.4 +httpx==0.28.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4a028ebb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +starlette==0.41.3 +uvicorn[standard]==0.32.1 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==23.1.0 diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..d96ae9d7 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,3 @@ +@echo off +python "%~dp0restart.py" +exit /b %ERRORLEVEL% diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..639651e8 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Restart API, PostgreSQL and Redis without removing named volumes.""" +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(["restart", "postgres", "redis", "api"]) + return subprocess.run(cmd, cwd=ROOT).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..81d27153 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +if [ -f .env ]; then + docker compose --env-file .env restart postgres redis api +else + docker compose restart postgres redis api +fi diff --git a/scripts/run_expiry.py b/scripts/run_expiry.py new file mode 100644 index 00000000..42d87200 --- /dev/null +++ b/scripts/run_expiry.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL.""" +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(args, extra_env=None): + env = os.environ.copy() + if extra_env: + env.update(extra_env) + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, env=env) + if proc.returncode != 0: + raise SystemExit(proc.returncode) + + +def wait_ready(): + import urllib.request + + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) + return + except Exception: + time.sleep(0.5) + raise SystemExit("API did not become ready") + + +def main(): + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"}) + wait_ready() + env = os.environ.copy() + env["SESSION_TTL_SECONDS"] = "2" + env["RUN_EXPIRY"] = "1" + proc = subprocess.run( + [sys.executable, "-m", "pytest", "student_tests/test_expiry.py", "-q"], + cwd=ROOT, + env=env, + ) + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"}) + wait_ready() + raise SystemExit(proc.returncode) + + +if __name__ == "__main__": + main() diff --git a/solve.md b/solve.md new file mode 100644 index 00000000..2e9423c6 --- /dev/null +++ b/solve.md @@ -0,0 +1,196 @@ +# Что происходит в лабораторной 2 + +Lab1 доказала, что контракт можно реализовать процессом в контейнере. Состояние жило в RAM и исчезало вместе с процессом. Lab2 решает другую задачу: **подтверждённые данные и ещё действующая сессия должны пережить рестарт**, а клиент должен быть **аутентифицированным пользователем**, а не строкой `X-User-Id`, которую любой может подставить. + +Целевая оценка этой работы — **5**, трек **согласованность кэша**. Ниже — не перечень файлов, а причины решений. + +--- + +## 1. Зачем вообще уходить из памяти + +In-memory store в lab1 был правильным минимумом: нет внешних зависимостей, легко гонять smoke. Он ломается в тот момент, когда появляется требование «после `docker compose restart` токен и сообщения на месте». + +Память процесса не является хранилищем: + +- SIGTERM / OOM / `restart` обнуляют словари; +- второй экземпляр API (это уже lab3) не увидит тех же пользователей; +- нельзя честно говорить про транзакции и индексы. + +Поэтому источник истины для пользователей, чатов, членства и сообщений — **PostgreSQL**. API становится тонким слоем: проверить контракт, открыть транзакцию, отдать JSON. Если Postgres недоступен, бизнес-путь нельзя обслуживать «из кэша как из правды»: кэш может быть неполным или устаревшим. + +SQLite сознательно не используется: стенд должен совпадать с тем, что будет масштабироваться дальше (сеть, пул, WAL, volume). + +--- + +## 2. Схема: ограничения в БД, а не только в Python + +Проверки в `app/validation.py` отсекают мусор на границе HTTP. Этого мало. Два клиента могут одновременно пройти проверку «username свободен» и оба сделать INSERT. Поэтому уникальность username — **UNIQUE INDEX в Postgres**. Гонка даёт `UniqueViolation` → один `201`, остальные `409`. Это не «оптимизация», а единственный способ получить корректность без распределённой блокировки на уровне приложения. + +Чат создаётся **атомарно**: строка `chats` и все строки `chat_members` в одной транзакции. Если среди `member_ids` есть несуществующий UUID, транзакция откатывается. Иначе получился бы «чат-сирота» без полного списка участников — объект, которого контракт не описывает и который нельзя честно отдать в `GET /chats`. + +Внешние ключи (`chat_members.user_id → users`, `messages.chat_id → chats`) держат инвариант даже если код ошибётся. `ON DELETE CASCADE` здесь учебный: обязательный API удаления не даёт, но схема не оставляет висящих членов при ручной чистке стенда. + +Порядок истории задан контрактом: `(created_at, id)` по возрастанию, keyset-cursor, `limit` 1–100. Запрос страницы: + +```sql +WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +ORDER BY created_at, id +LIMIT n +``` + +Индекс `messages (chat_id, created_at, id)` построен **под этот предикат**, а не «на всякий случай по chat_id». Без него Postgres читает и сортирует всю историю чата (seq scan). На 10 000 сообщений это видно в `EXPLAIN (ANALYZE, BUFFERS)`: Index Scan ~0.06 ms против Seq Scan ~2 ms. Пагинация не должна каждый раз сканировать всю таблицу — иначе стоимость чтения растёт линейно с возрастом чата, а cursor теряет смысл. + +Почему не `OFFSET`? `OFFSET 10000` всё равно просматривает 10 000 строк и отбрасывает их. Keyset опирается на уже известную точку `(created_at, id)` и идёт дальше по индексу. + +Cursor непрозрачный (base64 JSON + привязка к `chat_id`). Клиент не должен листать чужой чат, подставив cursor от другого. Это не криптография, а привязка курсора к ресурсу, как требует контракт. + +--- + +## 3. Почему сессия, а не `X-User-Id` + +В lab1 любой, кто знает UUID Алисы, *является* Алисой. Это учебный ярлык, не аутентификация. + +Новая модель: + +1. Регистрация принимает `password`. В таблицу попадает не пароль, а **Argon2id**. +2. Вход — только `POST /api/v1/auth/sessions` с `Authorization: Basic`. Basic здесь **кодирование** `username:password` в Base64, не шифрование. По HTTP на учебной машине это допустимо; на удалённой демонстрации нужен TLS (это lab7). +3. Ответ — opaque Bearer (≥256 бит случайности). Это не JWT и не username: сервер не парсит токен как утверждение, а **смотрит запись сессии**. +4. Остальные защищённые пути принимают только Bearer. Basic на `/users/me` — 401. `X-User-Id` игнорируется: иначе старый клиент lab1 обошёл бы новую модель. + +Logout удаляет ключ сессии. Повтор с тем же токеном — 401. TTL задаётся `SESSION_TTL_SECONDS`; рестарт **не продлевает** жизнь сессии. + +Токен в Redis хранится не как есть, а как `sess:`. Утечка дампа Redis не даёт сразу валидный Bearer (нужен прообраз). Это не замена TLS, но снижает ущерб от бэкапа volume. + +Если Redis недоступен в момент проверки Bearer, ответ **503**, не «ну нет сессии, пустим как анонима» и не 401 «неверный пароль». 401 означает «идентификация не принята». 503 означает «мы не можем её проверить». Путать их опасно: клиент при 401 выкинет токен, при 503 — повторит. + +--- + +## 4. Почему сессии в Redis, а не в Postgres + +Контракт разрешает оба варианта. Выбран Redis, потому что у сессии другая природа, чем у сообщения: + +- сессия — горячий ключ с TTL, частые GET на каждый запрос; +- сообщение — факт, который должен пережить годы и участвовать в JOIN/EXPLAIN; +- logout и expiry — естественный `DEL` / TTL ключа, без cron по таблице `sessions`. + +Проблема: **один TTL в памяти Redis не переживает рестарт**. Persistence-тест держит токен в памяти Python-процесса и перезапускает API **и хранилища**, volumes не трогая. Если сессии только в RAM Redis, после restart токен внезапно 401 — тест красный, хотя Postgres цел. + +Поэтому Redis запущен с **AOF** (`appendonly yes`, `appendfsync everysec`) и дополнительным RDB `save 60 1`. AOF после рестарта восстанавливает ключи **вместе с оставшимся TTL**. `everysec` — компромисс: можно потерять до ~1 с записей при crash процесса Redis, зато не платить `fsync` на каждый SET. Для учебных сессий это честная граница durability, её нужно называть на защите. + +Кэш истории живёт в том же Redis, но это **не источник истины**. Потеря кэша допустима: страница перечитывается из Postgres. Потеря сессий без AOF — уже нарушение контракта «действующая сессия переживает рестарт». + +--- + +## 5. Кэш истории и трек 5: какую ошибку мы чиним + +Контракт: после успешного POST сообщения следующее чтение через любую реплику должно его видеть. Кэш имеет право существовать, но **не имеет права отдать страницу старше успешной записи**. + +Наивный кэш `GET → Redis → иначе SQL → SET` ломается двумя способами. + +**Stale read.** Страница уже в Redis. POST пишет в Postgres и не трогает кэш. GET отдаёт старый `items` без нового сообщения. В эксперименте это воспроизводится флагом `invalidate=false`: так мы *показываем баг*, а не оставляем его в проде. Защита — версия `histver:`. Ключ страницы включает версию. После INSERT делается `INCR histver`. Старые ключи просто перестают попадать в lookup (и истекают по TTL). Не нужно SCAN/DEL всех страниц чата. + +**Cache stampede.** Истёк TTL или сменилась версия. 24 клиента одновременно делают miss. Без координации — 24 одинаковых `SELECT`. На горячем чате это бьёт в Postgres в тот момент, когда кэш как раз должен был его разгрузить. + +Singleflight: первый waiter запускает load, остальные ждут тот же `asyncio.Task` и получают тот же результат. Измерение: 24 SQL → 1 SQL при 24 конкурентных GET. Задержка 250 ms на load в lab-режиме нужна, чтобы HTTP-клиенты успели попасть в одно окно miss; без неё на localhost первый GET заполняет кэш раньше остальных, и stampede «не виден», хотя код всё ещё гоночный. + +Это in-process защита. Одна реплика lab2 — ок. На lab3 с двумя API понадобится lock в Redis (`SET NX`), иначе каждая реплика выполнит свой SQL. + +--- + +## 6. Инфраструктура: почему Compose и named volumes + +Skeleton даёт пустой `compose.example.yaml` — это список ролей, не стенд. Свой `compose.yaml` фиксирует: + +- версии образов, не `latest`; +- сеть только внутри compose: Postgres и Redis **не** публикуются на хост (API слушает `127.0.0.1:8080`); +- volumes `pg_data` / `redis_data` — данные переживают recreate контейнера; +- healthcheck: API не стартует, пока Postgres/Redis не `healthy`; `/health/ready` требует оба ping. + +`scripts/restart.sh` (на Windows — `restart.bat`) делает `docker compose restart postgres redis api` **без** `-v`. Persistence-тест не умеет отличить честный рестарт от no-op, поэтому в отчёте лежат `compose ps` до и после. + +Пул соединений ограничен (1–10) и с таймаутами: без этого при зависании Postgres запросы копятся бесконечно, а при всплеске легко исчерпать `max_connections`. + +Пароль — Argon2id, не SHA-256: SHA быстрый специально, его можно перебирать миллиардами попыток в секунду. Argon2 специально медленный и память-затратный. Параметры в этой работе учебные (`memory_cost=8192`), чтобы гонка 16 регистраций не убила контейнер на 512 MiB. В lab7 ждут более жёсткий Argon2id. + +--- + +## 7. Ошибки зависимостей + +| Ситуация | Код | Почему так | +| --- | --- | --- | +| Процесс жив, зависимости не важны | `/health/live` 200 | оркестратор не должен убивать процесс из-за рестарта Redis | +| Postgres или Redis не отвечают | `/health/ready` 503 | нельзя обещать безопасный синхронный путь | +| Bearer, Redis лежит | 503 | нельзя ни принять, ни отвергнуть сессию | +| Неверный пароль / нет токена | 401 + `WWW-Authenticate` | идентификация не удалась, сервис здоров | +| Чат чужой или не существует | 404 | 403 раскрыл бы факт существования чата | +| Username занят | 409 | конфликт уникальности, не «неверный запрос» | + +Сообщения ошибок не содержат SQL и stack trace. + +--- + +## Вопросы на защите (из README) + +### Что подтверждает COMMIT и какие гарантии зависят от настройки durability? + +`COMMIT` в PostgreSQL означает: транзакция завершена, изменения видны другим сессиям (с оговорками про уровень изоляции) и **записаны в WAL**. Это не магия «данные на диск навсегда». + +Что именно гарантировано, зависит от `synchronous_commit` и fsync: + +- по умолчанию `synchronous_commit=on`: клиент получает успех **после** сброса WAL на диск (на практике — в page cache ОС + политика fsync Postgres). После краша процесса Postgres транзакция, на которую API уже ответил 201, должна найтись после recovery. +- `synchronous_commit=off`: COMMIT возвращается раньше, можно потерять последние транзакции при crash — быстрее, слабее durability. +- Один `COMMIT` чата покрывает и чат, и членов: либо оба факта, либо ни одного. Это как раз граница транзакции в нашей схеме. + +Docker volume не равен репликации. `COMMIT` на одном диске ноутбука не спасает от `down -v` и от смерти диска. Persistence-тест доказывает другое: **контейнеры можно убить и поднять**, если volume на месте. + +Для Redis «COMMIT» в том же смысле нет. Есть запись в AOF. `appendfsync everysec` подтверждает SET клиенту, не дожидаясь fsync каждого ключа: окно потери ≈ 1 с. Это сознательная более слабая модель, чем у Postgres, и поэтому **сообщения не кладут только в Redis**. + +### Где хранятся сессии и что будет после flush/restart Redis? + +Сессия: ключ `sess:` в Redis, TTL = `SESSION_TTL_SECONDS`, payload `{user_id, expires_at}`. Пользователь, чат, текст сообщения — в Postgres. + +- **Штатный restart Redis с volume и AOF:** ключи и оставшийся TTL поднимаются. Persistence-тест и `test_redis_restart_keeps_session_and_history` это показывают. Кэш истории холодный, страницы читаются из Postgres, контракт read-after-write не ломается. +- **`FLUSHALL` / удаление volume Redis:** все сессии исчезают. Прежний Bearer → 401. Сообщения в Postgres на месте, но клиент должен войти заново. Для обязательного сценария lab2 это уже провал, если Redis — единственное хранилище сессий и persistence не включён. +- **Истёк TTL:** ключ удаляется самим Redis. Рестарт тут ни при чём и **не должен** оживлять сессию. + +Итог: Redis для сессий допустим только вместе с persistence, которая переживает проверяемый рестарт. Один TTL — это политика истечения, не сохранность. + +### Чем аутентификация отличается от доступа к конкретному чату? + +Два независимых решения, и контракт требует именно такого порядка: **сначала «кто ты», потом «можно ли этот объект»**. + +- **Аутентификация:** есть ли действительная сессия. Нет/истекла/отозвана/Redis не смогли проверить → 401 (или 503). На этом шаге чат ещё не смотрим. `X-User-Id` не аутентификация. +- **Авторизация на ресурс:** этот user_id есть в `chat_members` данного `chat_id`? Нет → **404**, не 403. 403 сказал бы клиенту: «чат существует, просто не твой». Для постороннего чат неотличим от несуществующего. + +Практический пример: Ева с валидным Bearer читает чат Алисы и Боба. Она аутентифицирована (не 401), но не член → 404. Список `GET /chats` отдаёт только её чаты. + +Создание чата: вызывающий обязан быть в `member_ids`. Это не «аутентификация», а правило состава объекта. Нарушение → 400 (некорректный список), не 401. + +### Почему hash пароля и быстрый checksum имеют разные задачи? + +Разные угрозы и разные желаемые свойства функции. + +**Password KDF (Argon2id).** Секрет, который человек помнит, короткий и угадываемый. Хеш лежит в БД. Если дамп утечёт, атакующий будет перебирать пароли. Нужна функция, которую **дорого** считать много раз: память, итерации, уникальная соль в каждом хеше. Быстрый SHA-256 здесь вреден: видеокарта перебирает миллиарды вариантов в секунду. Мы специально платим CPU на каждый login. + +**Checksum (SHA-256 файла, CRC, хеш токена в ключе Redis).** Цель — **обнаружить изменение или сделать короткий отпечаток**, не спрятать секрет от перебора. Нужна скорость и детерминизм. Для вложений в lab4 SHA-256 исходного PNG сверяет, что download = upload. Для ключа сессии SHA-256(token) — это lookup-индекс, не KDF: токен уже 256 бит случайности, его не подбирают по словарю как `qwerty123`. + +Путать их — типичная ошибка: «у нас пароли в SHA-256, как checksum файла». Checksum не задерживает атакующего. KDF не нужен, чтобы проверить целостность 10 MiB картинки. + +--- + +## Карта «проблема → решение» + +| Проблема | Решение | +| --- | --- | +| Рестарт API стирает чаты | Postgres + named volume | +| Два клиента занимают один username | UNIQUE + обработка UniqueViolation | +| Чат без части участников | Одна транзакция на chat+members | +| Пагинация сканирует всю историю | Составной индекс и keyset `(created_at, id)` | +| Клиент выдаёт себя за другого UUID | Opaque Bearer, `X-User-Id` игнорируется | +| Пароль в открытом виде / быстрый хеш | Argon2id | +| Сессия умирает вместе с Redis RAM | AOF + volume | +| GET после POST отдаёт старую страницу | Инвалидация `histver` | +| Толпа miss бьёт в одну и ту же SQL | Singleflight | +| Redis/Postgres лежат, а API врёт что готов | `/health/ready` 503; Bearer не downgrade в анонима | + +Это ровно переход lab1 → lab2: от «процесс умеет контракт» к «состояние и личность существуют отдельно от процесса и переживают его смерть». diff --git a/student_tests/__init__.py b/student_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/student_tests/conftest.py b/student_tests/conftest.py new file mode 100644 index 00000000..ae6a3291 --- /dev/null +++ b/student_tests/conftest.py @@ -0,0 +1,24 @@ +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +@pytest.fixture(scope="session") +def base_url(): + return os.environ.get("BASE_URL", "http://127.0.0.1:8080") + + +@pytest.fixture(scope="session") +def live(base_url): + c = Client(base_url, timeout=15) + try: + c.call("GET", "/health/ready", schema="Health") + except Failure as exc: + pytest.skip("API is not ready: %s" % exc) + return c diff --git a/student_tests/test_api.py b/student_tests/test_api.py new file mode 100644 index 00000000..60971ea1 --- /dev/null +++ b/student_tests/test_api.py @@ -0,0 +1,195 @@ +"""Negative, pagination and auth tests for lab 2.""" +import uuid +from urllib.parse import quote + +from client import Client + + +PASSWORD = "Example-only-not-a-real-secret-123" + + +def register(c: Client, username=None): + username = username or ("u_" + uuid.uuid4().hex[:10]) + user = c.call( + "POST", + "/api/v1/users", + 201, + body={"username": username, "display_name": "Test User", "password": PASSWORD}, + schema="User", + )["json"] + return user + + +def login(c: Client, username, password=PASSWORD): + import base64 + + cred = base64.b64encode((username + ":" + password).encode()).decode() + session = c.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + cred}, + schema="Session", + )["json"] + return {"id": session["user"]["id"], "user": session["user"], "token": session["token"], "registration": {"username": username, "password": password}} + + +def actor_of(c: Client, username=None): + user = register(c, username) + return login(c, user["username"]) + + +def test_malformed_json(live: Client): + response = live.call( + "POST", + "/api/v1/users", + 400, + raw=b"{not-json", + headers={"Content-Type": "application/json"}, + ) + assert response["json"]["error"]["code"] == "invalid_request" + assert response["headers"].get("X-Request-Id") + + +def test_duplicate_username(live: Client): + user = register(live, username="dup_" + uuid.uuid4().hex[:8]) + again = live.call( + "POST", + "/api/v1/users", + 409, + body={"username": user["username"], "display_name": "Other", "password": PASSWORD}, + ) + assert again["json"]["error"]["code"] == "conflict" + + +def test_empty_message_rejected(live: Client): + alice = actor_of(live) + bob = actor_of(live) + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + response = live.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=alice, + body={"text": " "}, + ) + assert response["status"] == 400 + + +def test_outsider_chat_is_404(live: Client): + alice = actor_of(live) + bob = actor_of(live) + eve = actor_of(live) + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "secret", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + response = live.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve) + assert response["json"]["error"]["code"] == "not_found" + + +def test_message_pagination_multiple_pages(live: Client): + alice = actor_of(live) + bob = actor_of(live) + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "pages", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + created = [] + for idx in range(5): + created.append(live.message(alice, chat, "m%s" % idx)["json"]) + collected = [] + cursor = None + for _ in range(10): + path = "/api/v1/chats/%s/messages?limit=2" % chat["id"] + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = live.call("GET", path, actor=bob, schema="MessagePage")["json"] + collected.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + assert len(collected) == 5 + ordered = sorted(created, key=lambda x: (x["created_at"], x["id"])) + assert collected == ordered + + +def test_cursor_from_other_chat_rejected(live: Client): + alice = actor_of(live) + bob = actor_of(live) + chat_a = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "a", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + chat_b = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "b", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + msg = live.message(alice, chat_a, "only in a")["json"] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=1" % chat_a["id"], + actor=alice, + schema="MessagePage", + )["json"] + assert page["items"][0]["id"] == msg["id"] + cursor = page["next_cursor"] + # If history is a single item, next_cursor is null; encode a foreign cursor via first page of A after a dummy extra message. + if cursor is None: + live.message(alice, chat_a, "second") + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=1" % chat_a["id"], + actor=alice, + schema="MessagePage", + )["json"] + cursor = page["next_cursor"] + response = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=10&cursor=%s" % (chat_b["id"], quote(cursor, safe="")), + 400, + actor=alice, + ) + assert response["status"] == 400 + + +def test_x_user_id_does_not_authenticate(live: Client): + alice = actor_of(live) + live.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + + +def test_wrong_password(live: Client): + user = register(live) + import base64 + + bad = base64.b64encode((user["username"] + ":wrong-password-xx").encode()).decode() + live.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad}) + + +def test_logout(live: Client): + alice = actor_of(live) + live.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + live.call("GET", "/api/v1/users/me", 401, actor=alice) diff --git a/student_tests/test_concurrent.py b/student_tests/test_concurrent.py new file mode 100644 index 00000000..5265031e --- /dev/null +++ b/student_tests/test_concurrent.py @@ -0,0 +1,48 @@ +"""10 clients × 10 messages = 100 unique messages.""" +import threading +import uuid + +from client import Client + +from student_tests.test_api import actor_of + + +def test_concurrent_messages(live: Client): + users = [actor_of(live) for _ in range(10)] + member_ids = [u["id"] for u in users] + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=users[0], + body={"title": "load", "member_ids": member_ids}, + schema="Chat", + )["json"] + + errors = [] + lock = threading.Lock() + + def worker(user): + local = Client(live.base, timeout=15) + for _ in range(10): + try: + local.message(user, chat, uuid.uuid4().hex) + except Exception as exc: + with lock: + errors.append(str(exc)) + + threads = [threading.Thread(target=worker, args=(u,)) for u in users] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=60) + + assert not errors, "unexpected failures: %s" % errors[:5] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=users[0], + schema="MessagePage", + )["json"] + ids = [item["id"] for item in page["items"]] + assert len(ids) == len(set(ids)) == 100 diff --git a/student_tests/test_expiry.py b/student_tests/test_expiry.py new file mode 100644 index 00000000..da31f3c9 --- /dev/null +++ b/student_tests/test_expiry.py @@ -0,0 +1,17 @@ +import os +import time + +import pytest +from client import Client + + +@pytest.mark.expiry +def test_session_expires(live: Client): + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + if os.environ.get("RUN_EXPIRY") != "1": + pytest.skip("set RUN_EXPIRY=1 and SESSION_TTL_SECONDS<=3") + assert ttl <= 3 + actor = live.user("ttl") + live.call("GET", "/api/v1/users/me", actor=actor, schema="User") + time.sleep(ttl + 1.5) + live.call("GET", "/api/v1/users/me", 401, actor=actor) diff --git a/student_tests/test_grade4.py b/student_tests/test_grade4.py new file mode 100644 index 00000000..92ceb895 --- /dev/null +++ b/student_tests/test_grade4.py @@ -0,0 +1,44 @@ +"""Grade-4: username race and atomic chat rollback.""" +import threading +import uuid + +from client import Client + +from student_tests.test_api import PASSWORD, actor_of + + +def test_register_race(live: Client): + username = "r_" + uuid.uuid4().hex[:12] + body = {"username": username, "display_name": "Race", "password": PASSWORD} + results = [] + + def once(): + local = Client(live.base, timeout=20) + try: + results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]) + except Exception as exc: + results.append(str(exc)) + + threads = [threading.Thread(target=once) for _ in range(16)] + for t in threads: + t.start() + for t in threads: + t.join() + assert results.count(201) == 1, results + assert results.count(409) == 15, results + + +def test_chat_rollback(live: Client): + alice = actor_of(live) + before = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + missing = str(uuid.uuid4()) + live.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]}, + ) + after = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + assert len(after) == len(before) + assert all(x["title"] != "rollback-should-not-exist" for x in after) diff --git a/student_tests/test_infra.py b/student_tests/test_infra.py new file mode 100644 index 00000000..fe1a022e --- /dev/null +++ b/student_tests/test_infra.py @@ -0,0 +1,89 @@ +"""Dependency failure and Redis restart (AOF + cache rebuild).""" +import os +import subprocess +import time +from pathlib import Path + +import pytest +from client import Client, Failure + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args): + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + if proc.returncode != 0: + raise AssertionError("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:])) + return proc + + +def wait_ready(c: Client, seconds=90): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = exc + time.sleep(0.5) + raise AssertionError("ready not restored: %s" % last) + + +def wait_status(c: Client, path, expected, seconds=30, actor=None): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + schema = "Health" if expected == 200 and path.startswith("/health") else None + c.call("GET", path, expected, actor=actor, schema=schema) + return + except Failure as exc: + last = exc + time.sleep(0.4) + raise AssertionError("did not observe %s for %s: %s" % (expected, path, last)) + + +@pytest.mark.infra +def test_redis_and_postgres_unavailable(live: Client): + if os.environ.get("RUN_INFRA") != "1": + pytest.skip("set RUN_INFRA=1 to stop compose services") + alice = live.user("infra") + compose("stop", "redis") + wait_status(live, "/health/live", 200) + wait_status(live, "/health/ready", 503) + wait_status(live, "/api/v1/users/me", 503, actor=alice) + compose("start", "redis") + wait_ready(live) + live.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + compose("stop", "postgres") + wait_status(live, "/health/live", 200) + wait_status(live, "/health/ready", 503) + compose("start", "postgres") + wait_ready(live) + live.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + +@pytest.mark.infra +def test_redis_restart_keeps_session_and_history(live: Client): + if os.environ.get("RUN_INFRA") != "1": + pytest.skip("set RUN_INFRA=1 to restart redis") + alice, bob, _eve, chat = live.bootstrap() + msg = live.message(alice, chat, "before-redis-restart")["json"] + compose("restart", "redis") + wait_ready(live) + me = live.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"] + assert me["id"] == alice["id"] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=bob, + schema="MessagePage", + )["json"] + assert any(m["id"] == msg["id"] for m in page["items"]) diff --git a/student_tests/test_track1.py b/student_tests/test_track1.py new file mode 100644 index 00000000..d71c3192 --- /dev/null +++ b/student_tests/test_track1.py @@ -0,0 +1,83 @@ +"""Track 1: cache consistency — stampede, stale read, read-after-write.""" +import json +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor + +from client import Client + + +def _lab_metrics(c: Client): + return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode()) + + +def _lab_flags(c: Client, body): + return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode()) + + +def test_read_after_write(live: Client): + alice, bob, _eve, chat = live.bootstrap() + _lab_flags(live, {"invalidate": True, "singleflight": True, "reset_sql": True}) + first = live.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=bob, + schema="MessagePage", + )["json"] + assert any(m["id"] == first["id"] for m in page["items"]) + + +def test_stale_without_invalidation_then_fixed(live: Client): + alice, bob, _eve, chat = live.bootstrap() + path = "/api/v1/chats/%s/messages?limit=50" % chat["id"] + live.call("GET", path, actor=alice, schema="MessagePage") + _lab_flags(live, {"invalidate": False, "singleflight": True}) + try: + stale_text = "stale-probe-" + uuid.uuid4().hex + live.message(alice, chat, stale_text) + page = live.call("GET", path, actor=bob, schema="MessagePage")["json"] + assert all(m["text"] != stale_text for m in page["items"]) + finally: + _lab_flags(live, {"invalidate": True}) + fresh_text = "fresh-probe-" + uuid.uuid4().hex + live.message(alice, chat, fresh_text) + page2 = live.call("GET", path, actor=bob, schema="MessagePage")["json"] + assert any(m["text"] == fresh_text for m in page2["items"]) + + +def test_stampede_singleflight_reduces_sql(live: Client): + alice, _bob, _eve, chat = live.bootstrap() + live.message(alice, chat, "stampede-seed") + path = "/api/v1/chats/%s/messages?limit=50" % chat["id"] + + def burst(n=24): + barrier = threading.Barrier(n) + + def one(_i): + local = Client(live.base) + barrier.wait() + return local.call("GET", path, actor=alice, schema="MessagePage") + + with ThreadPoolExecutor(max_workers=n) as pool: + list(pool.map(one, range(n))) + + try: + _lab_flags(live, {"singleflight": False, "invalidate": True, "reset_sql": True, "delay_ms": 250}) + live.message(alice, chat, "miss-" + uuid.uuid4().hex) + _lab_flags(live, {"reset_sql": True, "singleflight": False, "delay_ms": 250}) + burst() + unprotected = _lab_metrics(live)["history_sql"] + + _lab_flags(live, {"singleflight": True, "reset_sql": True, "delay_ms": 250}) + live.message(alice, chat, "miss2-" + uuid.uuid4().hex) + _lab_flags(live, {"reset_sql": True, "singleflight": True, "delay_ms": 250}) + burst() + protected = _lab_metrics(live)["history_sql"] + finally: + _lab_flags(live, {"invalidate": True, "singleflight": True, "delay_ms": 0}) + + print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected})) + assert unprotected >= 8, unprotected + assert protected < unprotected, (protected, unprotected) + assert protected <= 4, protected -- 2.54.0