diff --git a/.env.example b/.env.example index 714c807..e065c37 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,16 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 +CACHE_TTL_SECONDS=60 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url + +# Compose interpolates these. Synthetic local values only; replace after copy. +POSTGRES_PASSWORD=lab2-dev-only-change-me +CURSOR_SIGNING_KEY=lab2-dev-cursor-key-change-me-32bytes +# Application also accepts DATABASE_URL / REDIS_URL (compose injects them). +DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 diff --git a/.github/mtusi2026lab2.lnk b/.github/mtusi2026lab2.lnk new file mode 100644 index 0000000..5cfc18e Binary files /dev/null and b/.github/mtusi2026lab2.lnk differ diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289..7e020ea 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,5 +12,27 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + application: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: cp .env.example .env + - run: docker compose --env-file .env up -d --build + - name: Wait until API is ready + run: | + for i in $(seq 1 60); do + if curl -fsS http://127.0.0.1:8080/health/ready; then exit 0; fi + sleep 2 + done + docker compose --env-file .env logs + exit 1 + - run: make test PYTHON=python + - run: chmod +x scripts/restart.sh + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python + - run: python student-tests/run.py + - run: python student-tests/infra.py + - run: python scripts/run_expiry.py + - run: docker compose --env-file .env exec -T api python -m app.tools.explain diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1b53788 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12.8-slim-bookworm + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser \ + && apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY src/app /app/app +COPY migrations /app/migrations + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/app + +USER appuser +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \ + CMD curl -fsS http://127.0.0.1:8080/health/ready || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Makefile b/Makefile index 9ecb223..bc3e238 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence test-student test-infra test-expiry explain help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,3 +24,15 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-student: + $(PYTHON) student-tests/run.py + +test-infra: + $(PYTHON) student-tests/infra.py + +test-expiry: + $(PYTHON) scripts/run_expiry.py + +explain: + docker compose --env-file .env exec -T api python -m app.tools.explain diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..0addf26 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,78 @@ +services: + api: + build: + context: . + dockerfile: Dockerfile + image: mtusi-lab2-api:1.2.0 + ports: + - "8080:8080" + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60} + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + REDIS_URL: redis://redis:6379/0 + CURSOR_SIGNING_KEY: ${CURSOR_SIGNING_KEY} + DB_POOL_MIN: "1" + DB_POOL_MAX: "10" + DB_COMMAND_TIMEOUT: "5" + DB_CONNECT_TIMEOUT: "5" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + mem_limit: 512m + cpus: "1.0" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 20s + + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: messenger + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 512m + cpus: "1.0" + + redis: + image: redis:7.4.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --save + - "60" + - "1" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 128m + cpus: "0.50" + +volumes: + pg_data: + redis_data: diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..12b8862 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,51 @@ +-- Lab2 schema. PostgreSQL is the source of truth for users, chats, members, messages. +-- Transaction boundary: create-chat inserts the chat row and all members in one transaction. +-- Sessions live in Redis (AOF); this database does not store session tokens. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE UNIQUE INDEX users_username_key ON users (username); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id); + +-- List chats of a member ordered by (created_at, id). +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset pagination: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n +-- This composite index avoids a sequential scan of the whole history per page. +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d5ab258 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +fastapi==0.115.6 +uvicorn==0.32.1 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==23.1.0 +starlette==0.41.3 +pydantic==2.10.3 +python-multipart==0.0.20 diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 0000000..fb4c78d --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Restart API, PostgreSQL and Redis without removing named volumes.""" +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(["restart", "postgres", "redis", "api"]) + proc = subprocess.run(cmd, cwd=ROOT) + return proc.returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..81d2715 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +if [ -f .env ]; then + docker compose --env-file .env restart postgres redis api +else + docker compose restart postgres redis api +fi diff --git a/scripts/run_expiry.py b/scripts/run_expiry.py new file mode 100644 index 0000000..a5f51c4 --- /dev/null +++ b/scripts/run_expiry.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL.""" +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(args, extra_env=None): + env = os.environ.copy() + if extra_env: + env.update(extra_env) + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, env=env) + if proc.returncode != 0: + raise SystemExit(proc.returncode) + + +def wait_ready(): + import urllib.request + + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) + return + except Exception: + time.sleep(0.5) + raise SystemExit("API did not become ready") + + +def main(): + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"}) + wait_ready() + env = os.environ.copy() + env["SESSION_TTL_SECONDS"] = "2" + env["RUN_EXPIRY"] = "1" + proc = subprocess.run([sys.executable, str(ROOT / "student-tests" / "expiry.py")], cwd=ROOT, env=env) + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"}) + wait_ready() + raise SystemExit(proc.returncode) + + +if __name__ == "__main__": + main() diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 0000000..045965c --- /dev/null +++ b/src/app/__init__.py @@ -0,0 +1 @@ +"""MTUSI messenger API, laboratory 2.""" diff --git a/src/app/cache.py b/src/app/cache.py new file mode 100644 index 0000000..2dc8bd5 --- /dev/null +++ b/src/app/cache.py @@ -0,0 +1,133 @@ +import asyncio +import json +from collections.abc import Awaitable, Callable +from typing import TypeVar + +from . import rediscli +from .config import settings +from .errors import AppError + +T = TypeVar("T") + +CFG_SINGLEFLIGHT = "cfg:singleflight" +CFG_INVALIDATE = "cfg:invalidate" +SQL_HISTORY = "metrics:sql:messages_select" + + +class SingleFlight: + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._inflight: dict[str, asyncio.Task] = {} + + async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T: + async with self._lock: + task = self._inflight.get(key) + if task is None: + task = asyncio.create_task(factory()) + self._inflight[key] = task + owner = True + else: + owner = False + try: + return await asyncio.shield(task) + finally: + if owner: + async with self._lock: + self._inflight.pop(key, None) + + +_sf = SingleFlight() + + +async def _flag(name: str, default: bool = True) -> bool: + if not settings.lab_mode: + return default + try: + raw = await rediscli.require().get(name) + except Exception: + return default + if raw is None: + return default + return raw not in {"0", "false", "off"} + + +async def singleflight_enabled() -> bool: + return await _flag(CFG_SINGLEFLIGHT, True) + + +async def invalidate_enabled() -> bool: + return await _flag(CFG_INVALIDATE, True) + + +async def incr_history_sql() -> None: + try: + await rediscli.require().incr(SQL_HISTORY) + except Exception: + pass + + +async def history_sql_count() -> int: + try: + raw = await rediscli.require().get(SQL_HISTORY) + return int(raw or 0) + except Exception: + return 0 + + +async def reset_history_sql_count() -> None: + try: + await rediscli.require().set(SQL_HISTORY, 0) + except Exception: + pass + + +async def set_flags(*, singleflight: bool | None = None, invalidate: bool | None = None) -> dict: + r = rediscli.require() + if singleflight is not None: + await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0") + if invalidate is not None: + await r.set(CFG_INVALIDATE, "1" if invalidate else "0") + return { + "singleflight": await singleflight_enabled(), + "invalidate": await invalidate_enabled(), + "history_sql": await history_sql_count(), + } + + +def _page_key(chat_id: str, version: str, cursor: str, limit: int) -> str: + return f"hist:{chat_id}:{version}:{cursor}:{limit}" + + +async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict: + r = rediscli.require() + try: + version = await r.get("histver:" + chat_id) or "0" + key = _page_key(chat_id, version, cursor, limit) + cached = await r.get(key) + if cached: + return json.loads(cached) + + async def load_and_store() -> dict: + page = await loader() + try: + await r.set(key, json.dumps(page, ensure_ascii=False), ex=settings.cache_ttl_seconds) + except Exception: + pass + return page + + if await singleflight_enabled(): + return await _sf.do(key, load_and_store) + return await load_and_store() + except AppError: + raise + except Exception: + return await loader() + + +async def invalidate_history(chat_id: str) -> None: + if not await invalidate_enabled(): + return + try: + await rediscli.require().incr("histver:" + chat_id) + except Exception: + pass diff --git a/src/app/config.py b/src/app/config.py new file mode 100644 index 0000000..f7f4ed5 --- /dev/null +++ b/src/app/config.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path + + +def _file_or_env(name: str, default: str = "") -> str: + path = os.environ.get(name + "_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get(name, default) + + +class Settings: + def __init__(self) -> None: + self.app_host = os.environ.get("APP_HOST", "0.0.0.0") + self.app_port = int(os.environ.get("APP_PORT", "8080")) + self.app_env = os.environ.get("APP_ENV", "lab") + self.session_ttl_seconds = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + self.cache_ttl_seconds = int(os.environ.get("CACHE_TTL_SECONDS", "60")) + self.database_url = _file_or_env("DATABASE_URL") + self.redis_url = _file_or_env("REDIS_URL", "redis://redis:6379/0") + self.cursor_signing_key = _file_or_env("CURSOR_SIGNING_KEY", "lab2-dev-cursor-key-change-me") + self.db_pool_min = int(os.environ.get("DB_POOL_MIN", "1")) + self.db_pool_max = int(os.environ.get("DB_POOL_MAX", "10")) + self.db_command_timeout = float(os.environ.get("DB_COMMAND_TIMEOUT", "5")) + self.db_connect_timeout = float(os.environ.get("DB_CONNECT_TIMEOUT", "5")) + self.redis_socket_timeout = float(os.environ.get("REDIS_SOCKET_TIMEOUT", "2")) + + @property + def lab_mode(self) -> bool: + return self.app_env == "lab" + + +settings = Settings() diff --git a/src/app/context.py b/src/app/context.py new file mode 100644 index 0000000..632762b --- /dev/null +++ b/src/app/context.py @@ -0,0 +1,8 @@ +from contextvars import ContextVar + +request_id_var: ContextVar[str] = ContextVar("request_id", default="") +sql_count_var: ContextVar[int] = ContextVar("sql_count", default=0) + + +def bump_sql(n: int = 1) -> None: + sql_count_var.set(sql_count_var.get() + n) diff --git a/src/app/cursors.py b/src/app/cursors.py new file mode 100644 index 0000000..c1715ce --- /dev/null +++ b/src/app/cursors.py @@ -0,0 +1,35 @@ +import base64 +import hashlib +import hmac +import json + +from .config import settings +from .errors import AppError + + +def _sign(payload: str) -> str: + key = settings.cursor_signing_key.encode("utf-8") + return hmac.new(key, payload.encode("utf-8"), hashlib.sha256).hexdigest()[:24] + + +def encode_cursor(data: dict) -> str: + payload = json.dumps(data, separators=(",", ":"), ensure_ascii=False) + raw = payload + "." + _sign(payload) + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_cursor(token: str, kind: str, chat_id: str | None = None) -> dict: + try: + padded = token + "=" * (-len(token) % 4) + raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8") + payload, _, sig = raw.partition(".") + if not sig or not hmac.compare_digest(sig, _sign(payload)): + raise ValueError("bad signature") + data = json.loads(payload) + if data.get("k") != kind: + raise ValueError("wrong kind") + if chat_id is not None and data.get("c") != chat_id: + raise ValueError("wrong chat") + return data + except Exception: + raise AppError(400, "invalid_request", "Некорректный cursor") from None diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 0000000..3f4e2d3 --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,98 @@ +import asyncpg + +from .config import settings +from .context import bump_sql +from .errors import AppError + +pool: asyncpg.Pool | None = None + + +async def connect() -> asyncpg.Pool: + global pool + if not settings.database_url: + raise RuntimeError("DATABASE_URL is not set") + pool = await asyncpg.create_pool( + dsn=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + command_timeout=settings.db_command_timeout, + timeout=settings.db_connect_timeout, + ) + return pool + + +async def close() -> None: + global pool + if pool is not None: + await pool.close() + pool = None + + +def require_pool() -> asyncpg.Pool: + if pool is None: + raise AppError(503, "unavailable", "Сервис временно недоступен") + return pool + + +async def ping() -> None: + async with require_pool().acquire() as conn: + bump_sql() + await conn.execute("SELECT 1") + + +_INTEGRITY = ( + asyncpg.UniqueViolationError, + asyncpg.ForeignKeyViolationError, + asyncpg.CheckViolationError, +) + + +async def fetch(sql: str, *args, count: bool = True): + try: + async with require_pool().acquire() as conn: + if count: + bump_sql() + return await conn.fetch(sql, *args) + except AppError: + raise + except _INTEGRITY: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +async def fetchrow(sql: str, *args, count: bool = True): + try: + async with require_pool().acquire() as conn: + if count: + bump_sql() + return await conn.fetchrow(sql, *args) + except AppError: + raise + except _INTEGRITY: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +async def execute(sql: str, *args, count: bool = True): + try: + async with require_pool().acquire() as conn: + if count: + bump_sql() + return await conn.execute(sql, *args) + except AppError: + raise + except _INTEGRITY: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +async def transaction_connection(): + try: + return require_pool().acquire() + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None diff --git a/src/app/errors.py b/src/app/errors.py new file mode 100644 index 0000000..f41964c --- /dev/null +++ b/src/app/errors.py @@ -0,0 +1,25 @@ +from fastapi.responses import JSONResponse + + +class AppError(Exception): + def __init__(self, status: int, code: str, message: str, www_authenticate: str | None = None) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.www_authenticate = www_authenticate + + +def error_response(status: int, code: str, message: str, request_id: str, www_authenticate: str | None = None) -> JSONResponse: + headers = {"X-Request-Id": request_id} + if www_authenticate: + headers["WWW-Authenticate"] = www_authenticate + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers, + ) + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 0000000..ad54f3f --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,76 @@ +from contextlib import asynccontextmanager +import uuid + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import db, rediscli +from .config import settings +from .context import request_id_var, sql_count_var +from .errors import AppError, error_response +from .migrate import apply_migrations +from .routes import api, health + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + await db.connect() + await apply_migrations(db.require_pool()) + await rediscli.connect() + yield + await rediscli.close() + await db.close() + + +app = FastAPI(title="MTUSI Messenger", version="1.2.0", lifespan=lifespan, redirect_slashes=False) +app.include_router(health) +app.include_router(api, prefix="/api/v1") + + +@app.middleware("http") +async def request_context(request: Request, call_next): + rid = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = rid + token_rid = request_id_var.set(rid) + token_sql = sql_count_var.set(0) + sql_count = 0 + try: + response = await call_next(request) + sql_count = sql_count_var.get() + finally: + request_id_var.reset(token_rid) + sql_count_var.reset(token_sql) + response.headers["X-Request-Id"] = rid + if settings.lab_mode: + response.headers["X-Sql-Count"] = str(sql_count) + return response + + +@app.exception_handler(AppError) +async def handle_app_error(request: Request, exc: AppError): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + return error_response(exc.status, exc.code, exc.message, rid, exc.www_authenticate) + + +@app.exception_handler(RequestValidationError) +async def handle_validation(request: Request, _exc: RequestValidationError): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + return error_response(400, "invalid_request", "Некорректный запрос", rid) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http(request: Request, exc: StarletteHTTPException): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + if exc.status_code == 404: + return error_response(404, "not_found", "Объект не найден", rid) + if exc.status_code == 405: + return error_response(400, "invalid_request", "Некорректный запрос", rid) + return error_response(exc.status_code, "internal_error", "Внутренняя ошибка", rid) + + +@app.exception_handler(Exception) +async def handle_unexpected(request: Request, _exc: Exception): + rid = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + return error_response(500, "internal_error", "Внутренняя ошибка", rid) diff --git a/src/app/migrate.py b/src/app/migrate.py new file mode 100644 index 0000000..9215e7c --- /dev/null +++ b/src/app/migrate.py @@ -0,0 +1,38 @@ +from pathlib import Path + +import asyncpg + + +def _migrations_dir() -> Path: + here = Path(__file__).resolve().parent + for candidate in (here.parent.parent / "migrations", Path("/app/migrations")): + if candidate.is_dir(): + return candidate + raise RuntimeError("migrations directory not found") + + +MIGRATIONS = _migrations_dir() + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + async with pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")} + files = sorted(p for p in MIGRATIONS.glob("*.sql") if p.is_file()) + for path in files: + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + "INSERT INTO schema_migrations(filename) VALUES($1)", + path.name, + ) diff --git a/src/app/passwords.py b/src/app/passwords.py new file mode 100644 index 0000000..8431b23 --- /dev/null +++ b/src/app/passwords.py @@ -0,0 +1,23 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + +hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1) +_dummy_hash = None + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def dummy_hash() -> str: + global _dummy_hash + if _dummy_hash is None: + _dummy_hash = hash_password("timing-dummy-password") + return _dummy_hash + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError, ValueError, TypeError): + return False diff --git a/src/app/rediscli.py b/src/app/rediscli.py new file mode 100644 index 0000000..e38c6b5 --- /dev/null +++ b/src/app/rediscli.py @@ -0,0 +1,44 @@ +import redis.asyncio as redis + +from .config import settings +from .errors import AppError + +client: redis.Redis | None = None + + +async def connect() -> redis.Redis: + global client + client = redis.from_url( + settings.redis_url, + socket_connect_timeout=settings.redis_socket_timeout, + socket_timeout=settings.redis_socket_timeout, + decode_responses=True, + ) + await client.ping() + return client + + +async def close() -> None: + global client + if client is not None: + await client.close() + client = None + + +def require() -> redis.Redis: + if client is None: + raise AppError(503, "unavailable", "Сервис временно недоступен") + return client + + +async def ping() -> None: + try: + await require().ping() + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +def wrap_redis_errors(exc: Exception) -> AppError: + return AppError(503, "unavailable", "Сервис временно недоступен") diff --git a/src/app/routes.py b/src/app/routes.py new file mode 100644 index 0000000..e80b9de --- /dev/null +++ b/src/app/routes.py @@ -0,0 +1,378 @@ +import base64 +import binascii +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import asyncpg +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse, Response +from pydantic import BaseModel, ConfigDict, Field + +from . import cache, db, rediscli, sessions +from .cursors import decode_cursor, encode_cursor +from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError +from .config import settings +from .passwords import dummy_hash, hash_password, verify_password +from .serialize import chat_json, message_json, user_json +from .timeutil import fmt_dt + +api = APIRouter() +health = APIRouter() + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(min_length=1, max_length=4000) + + +def _blank(value: str) -> bool: + return value.strip() == "" + + +def _limit(raw: str | None) -> int: + if raw is None or raw == "": + return 50 + try: + value = int(raw) + except ValueError: + raise AppError(400, "invalid_request", "Некорректный limit") from None + if value < 1 or value > 100: + raise AppError(400, "invalid_request", "Некорректный limit") + return value + + +async def current_user(request: Request) -> asyncpg.Record: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + user_id = await sessions.resolve(rest.strip()) + row = await db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id) + if row is None: + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + return row + + +async def member_ids_of(chat_id: UUID) -> list[UUID]: + rows = await db.fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id) + return [r["user_id"] for r in rows] + + +async def load_chat_for_member(chat_id: UUID, user_id: UUID) -> asyncpg.Record: + row = await db.fetchrow( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 + WHERE c.id = $1 + """, + chat_id, + user_id, + ) + if row is None: + raise AppError(404, "not_found", "Объект не найден") + return row + + +@health.get("/health/live") +async def live() -> JSONResponse: + return JSONResponse({"status": "ok"}) + + +@health.get("/health/ready") +async def ready() -> JSONResponse: + await db.ping() + await rediscli.ping() + return JSONResponse({"status": "ok"}) + + +@api.post("/users") +async def register(body: CreateUser) -> JSONResponse: + if _blank(body.display_name) or _blank(body.password) or _blank(body.username): + raise AppError(400, "invalid_request", "Некорректные поля") + user_id = uuid4() + created = datetime.now(timezone.utc) + try: + await db.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + user_id, + body.username, + body.display_name, + hash_password(body.password), + created, + ) + except asyncpg.UniqueViolationError: + raise AppError(409, "conflict", "Имя пользователя занято") from None + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + return JSONResponse( + user_json({"id": user_id, "username": body.username, "display_name": body.display_name, "created_at": created}), + status_code=201, + ) + + +@api.get("/users/me") +async def me(request: Request) -> JSONResponse: + row = await current_user(request) + return JSONResponse(user_json(row)) + + +@api.post("/auth/sessions") +async def login(request: Request) -> JSONResponse: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "basic" or not rest.strip(): + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) + try: + decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeError, ValueError): + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) from None + username, sep, password = decoded.partition(":") + if not sep: + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) + row = await db.fetchrow( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1", + username, + ) + password_hash = row["password_hash"] if row else dummy_hash() + if row is None or not verify_password(password_hash, password): + raise AppError(401, "unauthorized", "Требуется аутентификация", BASIC_CHALLENGE) + token, expires_at = await sessions.create(row["id"]) + return JSONResponse( + { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": user_json(row), + }, + status_code=201, + ) + + +@api.delete("/auth/sessions/current") +async def logout(request: Request) -> Response: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + await sessions.revoke(rest.strip()) + return Response(status_code=204) + + +@api.post("/chats") +async def create_chat(request: Request, body: CreateChat) -> JSONResponse: + if _blank(body.title): + raise AppError(400, "invalid_request", "Некорректные поля") + if len(body.member_ids) != len(set(body.member_ids)): + raise AppError(400, "invalid_request", "Некорректный список участников") + user = await current_user(request) + if user["id"] not in body.member_ids: + raise AppError(400, "invalid_request", "Некорректный список участников") + chat_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require_pool().acquire() as conn: + async with conn.transaction(): + existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", body.member_ids) + if len(existing) != len(body.member_ids): + raise AppError(400, "invalid_request", "Некорректный список участников") + await conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", + chat_id, + body.title, + created, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, mid) for mid in body.member_ids], + ) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + return JSONResponse(chat_json({"id": chat_id, "title": body.title, "created_at": created}, body.member_ids), status_code=201) + + +@api.get("/chats") +async def list_chats(request: Request) -> JSONResponse: + user = await current_user(request) + limit = _limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + args: list = [user["id"]] + clause = "" + if cursor: + data = decode_cursor(cursor, "chat") + clause = "AND (c.created_at, c.id) > ($2, $3)" + args.extend([datetime.fromisoformat(data["t"].replace("Z", "+00:00")), UUID(data["i"])]) + rows = await db.fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1 + WHERE 1=1 {clause} + ORDER BY c.created_at, c.id + LIMIT {limit + 1} + """, + *args, + ) + page = rows[:limit] + items = [] + for row in page: + items.append(chat_json(row, await member_ids_of(row["id"]))) + next_cursor = None + if len(rows) > limit: + last = page[-1] + next_cursor = encode_cursor({"k": "chat", "t": fmt_dt(last["created_at"]), "i": str(last["id"])}) + return JSONResponse({"items": items, "next_cursor": next_cursor}) + + +@api.get("/chats/{chat_id}") +async def get_chat(request: Request, chat_id: UUID) -> JSONResponse: + user = await current_user(request) + row = await load_chat_for_member(chat_id, user["id"]) + return JSONResponse(chat_json(row, await member_ids_of(chat_id))) + + +async def _load_messages(chat_id: UUID, after_ts: datetime | None, after_id: UUID | None, limit: int) -> dict: + await cache.incr_history_sql() + if after_ts is None: + rows = await db.fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id + LIMIT $2 + """, + chat_id, + limit + 1, + ) + else: + rows = await db.fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT $4 + """, + chat_id, + after_ts, + after_id, + limit + 1, + ) + page = rows[:limit] + items = [message_json(r) for r in page] + next_cursor = None + if len(rows) > limit: + last = page[-1] + next_cursor = encode_cursor( + {"k": "msg", "c": str(chat_id), "t": fmt_dt(last["created_at"]), "i": str(last["id"])} + ) + return {"items": items, "next_cursor": next_cursor} + + +@api.post("/chats/{chat_id}/messages") +async def create_message(request: Request, chat_id: UUID, body: CreateMessage) -> JSONResponse: + if _blank(body.text): + raise AppError(400, "invalid_request", "Некорректные поля") + user = await current_user(request) + await load_chat_for_member(chat_id, user["id"]) + message_id = uuid4() + created = datetime.now(timezone.utc) + try: + await db.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + message_id, + chat_id, + user["id"], + body.text, + created, + ) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + await cache.invalidate_history(str(chat_id)) + return JSONResponse( + message_json( + { + "id": message_id, + "chat_id": chat_id, + "sender_id": user["id"], + "text": body.text, + "created_at": created, + } + ), + status_code=201, + ) + + +@api.get("/chats/{chat_id}/messages") +async def list_messages(request: Request, chat_id: UUID) -> JSONResponse: + user = await current_user(request) + await load_chat_for_member(chat_id, user["id"]) + limit = _limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") or "" + after_ts = after_id = None + if cursor: + data = decode_cursor(cursor, "msg", str(chat_id)) + after_ts = datetime.fromisoformat(data["t"].replace("Z", "+00:00")) + after_id = UUID(data["i"]) + + async def loader(): + return await _load_messages(chat_id, after_ts, after_id, limit) + + page = await cache.get_history_page(str(chat_id), cursor, limit, loader) + return JSONResponse(page) + + +class LabFlags(BaseModel): + model_config = ConfigDict(extra="forbid") + singleflight: bool | None = None + invalidate: bool | None = None + reset_sql: bool | None = None + + +@api.get("/_lab/metrics") +async def lab_metrics() -> JSONResponse: + if not settings.lab_mode: + raise AppError(404, "not_found", "Объект не найден") + return JSONResponse( + { + "singleflight": await cache.singleflight_enabled(), + "invalidate": await cache.invalidate_enabled(), + "history_sql": await cache.history_sql_count(), + } + ) + + +@api.post("/_lab/flags") +async def lab_flags(body: LabFlags) -> JSONResponse: + if not settings.lab_mode: + raise AppError(404, "not_found", "Объект не найден") + if body.reset_sql: + await cache.reset_history_sql_count() + flags = await cache.set_flags(singleflight=body.singleflight, invalidate=body.invalidate) + return JSONResponse(flags) diff --git a/src/app/serialize.py b/src/app/serialize.py new file mode 100644 index 0000000..37f0371 --- /dev/null +++ b/src/app/serialize.py @@ -0,0 +1,31 @@ +from uuid import UUID + +from .timeutil import fmt_dt + + +def user_json(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": fmt_dt(row["created_at"]), + } + + +def chat_json(row, member_ids: list[UUID]) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(x) for x in member_ids], + "created_at": fmt_dt(row["created_at"]), + } + + +def message_json(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": fmt_dt(row["created_at"]), + } diff --git a/src/app/sessions.py b/src/app/sessions.py new file mode 100644 index 0000000..bb19b76 --- /dev/null +++ b/src/app/sessions.py @@ -0,0 +1,56 @@ +import hashlib +import json +import secrets +from datetime import timedelta +from uuid import UUID + +from . import rediscli +from .config import settings +from .errors import AppError, BEARER_CHALLENGE +from .timeutil import fmt_dt, now_utc + + +def new_token() -> str: + return secrets.token_urlsafe(32) + + +def _key(token: str) -> str: + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "sess:" + digest + + +async def create(user_id: UUID) -> tuple[str, str]: + token = new_token() + expires = now_utc() + timedelta(seconds=settings.session_ttl_seconds) + payload = json.dumps({"user_id": str(user_id), "expires_at": fmt_dt(expires)}) + try: + await rediscli.require().set(_key(token), payload, ex=settings.session_ttl_seconds) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + return token, fmt_dt(expires) + + +async def resolve(token: str) -> UUID: + try: + raw = await rediscli.require().get(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + if not raw: + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) + data = json.loads(raw) + return UUID(data["user_id"]) + + +async def revoke(token: str) -> None: + try: + deleted = await rediscli.require().delete(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + if not deleted: + raise AppError(401, "unauthorized", "Требуется аутентификация", BEARER_CHALLENGE) diff --git a/src/app/timeutil.py b/src/app/timeutil.py new file mode 100644 index 0000000..ff98eb6 --- /dev/null +++ b/src/app/timeutil.py @@ -0,0 +1,12 @@ +from datetime import datetime, timezone + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def fmt_dt(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + value = value.astimezone(timezone.utc) + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") diff --git a/src/app/tools/__init__.py b/src/app/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app/tools/explain.py b/src/app/tools/explain.py new file mode 100644 index 0000000..5178d4d --- /dev/null +++ b/src/app/tools/explain.py @@ -0,0 +1,85 @@ +"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for history pagination.""" +import asyncio +import os +from uuid import uuid4 + +import asyncpg + +from app.passwords import hash_password + + +async def main() -> None: + url = os.environ["DATABASE_URL"] + conn = await asyncpg.connect(dsn=url, timeout=10, command_timeout=120) + try: + alice = uuid4() + bob = uuid4() + chat = uuid4() + await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)", + alice, + "explain_alice_" + alice.hex[:8], + "Alice", + hash_password("Example-only-not-a-real-secret-123"), + bob, + "explain_bob_" + bob.hex[:8], + "Bob", + ) + await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat") + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)", + chat, + alice, + bob, + ) + n = int(os.environ.get("EXPLAIN_MESSAGES", "10000")) + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval + FROM generate_series(1, $3) AS g + """, + chat, + alice, + n, + ) + count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat) + print("seeded_messages", count, "chat_id", chat) + + query = """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT 50 + """ + first = await conn.fetchrow( + "SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1", + chat, + ) + print("\n=== WITH index messages_chat_created_id_idx ===") + plan = await conn.fetch( + "EXPLAIN (ANALYZE, BUFFERS) " + query, + chat, + first["created_at"], + first["id"], + ) + print("\n".join(row[0] for row in plan)) + + await conn.execute("SET enable_indexscan = off") + await conn.execute("SET enable_bitmapscan = off") + await conn.execute("SET enable_indexonlyscan = off") + print("\n=== WITHOUT index scans (seq scan baseline) ===") + plan2 = await conn.fetch( + "EXPLAIN (ANALYZE, BUFFERS) " + query, + chat, + first["created_at"], + first["id"], + ) + print("\n".join(row[0] for row in plan2)) + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/student-tests/expiry.py b/student-tests/expiry.py new file mode 100644 index 0000000..b242225 --- /dev/null +++ b/student-tests/expiry.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +os.environ["RUN_EXPIRY"] = "1" +from run import test_expiry # noqa: E402 + +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, run # noqa: E402 + + +def main(): + c = Client(os.environ.get("BASE_URL", "http://localhost:8080")) + test_expiry(c) + print("PASS: session expiry") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/infra.py b/student-tests/infra.py new file mode 100644 index 0000000..6a677d4 --- /dev/null +++ b/student-tests/infra.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Dependency failure and Redis-only restart checks (Docker Compose).""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run # noqa: E402 + + +def compose(*args): + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + if proc.returncode != 0: + raise Failure("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:])) + return proc + + +def wait_ready(c: Client, seconds=90): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = exc + time.sleep(0.5) + raise Failure("ready not restored: %s" % last) + + +def wait_status(c: Client, path, expected, seconds=30, actor=None): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + c.call("GET", path, expected, actor=actor, schema="Health" if expected == 200 and path.startswith("/health") else None) + return + except Failure as exc: + last = exc + time.sleep(0.4) + raise Failure("did not observe %s for %s: %s" % (expected, path, last)) + + +def main(): + base = os.environ.get("BASE_URL", "http://localhost:8080") + c = Client(base, timeout=3) + alice = c.user("infra") + compose("ps") + + compose("stop", "redis") + wait_status(c, "/health/live", 200) + wait_status(c, "/health/ready", 503) + wait_status(c, "/api/v1/users/me", 503, actor=alice) + compose("start", "redis") + wait_ready(c) + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + compose("stop", "postgres") + wait_status(c, "/health/live", 200) + wait_status(c, "/health/ready", 503) + compose("start", "postgres") + wait_ready(c) + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + alice2, bob, _eve, chat = c.bootstrap() + msg = c.message(alice2, chat, "before-redis-restart")["json"] + compose("restart", "redis") + wait_ready(c) + me = c.call("GET", "/api/v1/users/me", actor=alice2, schema="User")["json"] + check(me["id"] == alice2["id"], "session lost after Redis restart despite AOF") + page = c.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=bob, + schema="MessagePage", + )["json"] + check(any(m["id"] == msg["id"] for m in page["items"]), "history not rebuilt after cache restart") + print(json.dumps({"redis_restart": "session+history ok", "db_or_redis_down": "503"}, ensure_ascii=False)) + print("PASS: infra unavailability and redis restart") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 0000000..9117863 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Student checks for grades 4 and 5 (track 1: cache consistency).""" +import base64 +import json +import os +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run # noqa: E402 + + +def _basic(username, password): + return "Basic " + base64.b64encode((username + ":" + password).encode()).decode() + + +def lab_metrics(c: Client): + return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode()) + + +def lab_flags(c: Client, body): + return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode()) + + +def test_wrong_password_and_x_user_id(c: Client): + actor = c.user("neg") + bad = _basic(actor["registration"]["username"], "wrong-password-xx") + c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad}) + c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": actor["id"]}) + c.call("GET", "/api/v1/users/me", 401) + + +def test_logout(c: Client): + actor = c.user("out") + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor) + c.call("GET", "/api/v1/users/me", 401, actor=actor) + + +def test_register_race(c: Client): + username = "r_" + uuid.uuid4().hex[:12] + body = { + "username": username, + "display_name": "Race", + "password": "Example-only-not-a-real-secret-123", + } + results = [] + + def once(): + local = Client(c.base) + try: + results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]) + except Failure as exc: + results.append(str(exc)) + + threads = [threading.Thread(target=once) for _ in range(16)] + for t in threads: + t.start() + for t in threads: + t.join() + created = results.count(201) + conflicted = results.count(409) + check(created == 1, "username race: expected exactly one 201, got %s" % results) + check(conflicted == 15, "username race: expected fifteen 409, got %s" % results) + + +def test_chat_rollback(c: Client): + alice = c.user("rb") + before = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + missing = str(uuid.uuid4()) + c.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]}, + ) + after = c.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + check(len(after) == len(before), "composite chat create leaked a row after rollback") + check(all(x["title"] != "rollback-should-not-exist" for x in after), "rolled-back title became visible") + + +def test_read_after_write(c: Client): + alice, bob, _eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + lab_flags(c, {"invalidate": True, "singleflight": True, "reset_sql": True}) + first = c.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"] + page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + check(any(m["id"] == first["id"] for m in page["items"]), "read-after-write lost the new message") + + +def test_stale_without_invalidation_then_fixed(c: Client): + alice, bob, _eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50" + c.call("GET", path, actor=alice, schema="MessagePage") + lab_flags(c, {"invalidate": False, "singleflight": True}) + stale_text = "stale-probe-" + uuid.uuid4().hex + c.message(alice, chat, stale_text) + page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + stale = all(m["text"] != stale_text for m in page["items"]) + check(stale, "expected a stale cache read when invalidation is disabled") + lab_flags(c, {"invalidate": True}) + fresh_text = "fresh-probe-" + uuid.uuid4().hex + c.message(alice, chat, fresh_text) + page2 = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + check(any(m["text"] == fresh_text for m in page2["items"]), "invalidation did not restore read-after-write") + + +def test_stampede(c: Client): + alice, _bob, _eve, chat = c.bootstrap() + c.message(alice, chat, "stampede-seed") + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50" + + def burst(n=24): + barrier = threading.Barrier(n) + + def one(_i): + local = Client(c.base) + barrier.wait() + return local.call("GET", path, actor=alice, schema="MessagePage") + + with ThreadPoolExecutor(max_workers=n) as pool: + list(pool.map(one, range(n))) + + lab_flags(c, {"singleflight": False, "invalidate": True, "reset_sql": True}) + c.message(alice, chat, "miss-" + uuid.uuid4().hex) + lab_flags(c, {"reset_sql": True, "singleflight": False}) + burst() + unprotected = lab_metrics(c)["history_sql"] + + lab_flags(c, {"singleflight": True, "reset_sql": True}) + c.message(alice, chat, "miss2-" + uuid.uuid4().hex) + lab_flags(c, {"reset_sql": True, "singleflight": True}) + burst() + protected = lab_metrics(c)["history_sql"] + print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected}, ensure_ascii=False)) + check(unprotected >= 8, "unprotected stampede did not generate enough SQL") + check(protected < unprotected, "singleflight did not reduce SQL during stampede") + check(protected <= 4, "singleflight still issued too many history SQL queries") + + +def test_expiry(c: Client): + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + check(ttl <= 3, "expiry test requires SESSION_TTL_SECONDS<=3 (recreate api with short TTL)") + actor = c.user("ttl") + c.call("GET", "/api/v1/users/me", actor=actor, schema="User") + time.sleep(ttl + 1.5) + c.call("GET", "/api/v1/users/me", 401, actor=actor) + + +def main(): + base = os.environ.get("BASE_URL", "http://localhost:8080") + c = Client(base) + c.call("GET", "/health/live", schema="Health") + c.call("GET", "/health/ready", schema="Health") + tests = [ + test_wrong_password_and_x_user_id, + test_logout, + test_register_race, + test_chat_rollback, + test_read_after_write, + test_stale_without_invalidation_then_fixed, + test_stampede, + ] + if os.environ.get("RUN_EXPIRY") == "1": + tests.append(test_expiry) + try: + for fn in tests: + fn(c) + print("PASS", fn.__name__) + finally: + try: + lab_flags(c, {"invalidate": True, "singleflight": True}) + except Exception: + pass + print("PASS: student grade 4/5 HTTP checks") + + +if __name__ == "__main__": + run(main)