From 0f8dc7a6e478b2fce0b5f4055c717dc49bb862bd Mon Sep 17 00:00:00 2001 From: "yurtaev.va" Date: Fri, 23 Oct 2026 18:39:11 +0300 Subject: [PATCH 1/2] feat: second lab --- .env.example | 12 +- Dockerfile | 9 + Makefile | 5 +- REPORT.md | 73 +++-- app/__init__.py | 0 app/main.py | 475 ++++++++++++++++++++++++++++ compose.yaml | 52 +++ evidence/cache_experiment_run1.json | 63 ++++ evidence/cache_experiment_run2.json | 63 ++++ evidence/cache_experiment_run3.json | 63 ++++ evidence/history_explain.txt | 15 + evidence/load.json | 17 + migrations/001_initial.sql | 38 +++ requirements.txt | 4 + scripts/cache_experiment.py | 116 +++++++ scripts/lint.py | 49 +++ scripts/restart.sh | 5 + tests/level4.py | 121 +++++++ 18 files changed, 1151 insertions(+), 29 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 compose.yaml create mode 100644 evidence/cache_experiment_run1.json create mode 100644 evidence/cache_experiment_run2.json create mode 100644 evidence/cache_experiment_run3.json create mode 100644 evidence/history_explain.txt create mode 100644 evidence/load.json create mode 100644 migrations/001_initial.sql create mode 100644 requirements.txt create mode 100644 scripts/cache_experiment.py create mode 100644 scripts/lint.py create mode 100755 scripts/restart.sh create mode 100644 tests/level4.py diff --git a/.env.example b/.env.example index 714c8076..f55e7768 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,10 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# Set a local synthetic password in .env; never commit it. +POSTGRES_PASSWORD=replace-me-with-a-local-password +DB_POOL_SIZE=10 +CACHE_ENABLED=1 +CACHE_LOCK_ENABLED=1 +CACHE_MISS_DELAY_MS=0 +ENABLE_EXPERIMENT_METRICS= diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e8e79c31 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12.10-slim +WORKDIR /app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt && useradd -r -u 10001 app +COPY app ./app +COPY migrations ./migrations +USER app +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"] diff --git a/Makefile b/Makefile index 9ecb223e..b0e423a1 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate lint test load test-persistence help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -15,6 +15,9 @@ check: validate: $(PYTHON) scripts/validate_openapi.py +lint: + $(PYTHON) scripts/lint.py + test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index e42f2b05..d2a95b07 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,64 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: Юртаев (группа не указана). Цель: **5, трек 1 — согласованность кэша**. Commit/PR не создавались. Использованы Python 3.12, FastAPI, psycopg 3, redis-py, `hashlib.scrypt`; реализация и эксперимент подготовлены с помощью Codex. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверено на Apple M4, 16 GiB RAM, Docker 29.3.0. Образы: Python 3.12.10, PostgreSQL 16.8, Redis 7.4.2. Явных CPU/RAM лимитов нет; Docker Desktop показывал 7.65 GiB доступной памяти. -## Архитектура и решения +```sh +cp .env.example .env +# Заменить POSTGRES_PASSWORD в локальном .env на случайный учебный пароль. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +make lint check test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/level4.py +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +API открыт только на `127.0.0.1:8080`. При первом старте миграции `migrations/*.sql` применяются под транзакционной advisory lock; повторный старт их пропускает. Остановка: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. `.env` и volume исключены из Git. -## Доказательства критериев +## Архитектура -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Из первой лабораторной сохранены пользователи, чаты, сообщения, проверки доступа и keyset pagination. Идентификация изменена по контракту: регистрация с паролем, Basic только для создания сессии, Bearer для остальных защищённых endpoint. PostgreSQL — источник истины для users, chats, chat_members, messages, sessions и chat_versions. Redis содержит только страницы истории (TTL 30 с) и краткие lock-ключи; его данные можно потерять. Сессии в PostgreSQL переживают полный рестарт, не продлевают TTL и отзываются удалением. В БД хранится SHA-256 случайного 256-битного токена как индексный идентификатор; пароль хешируется `scrypt` с отдельной солью 128 бит и параметрами `N=16384,r=8,p=1`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Схема задаёт UUID PK, FK, уникальный username, PK `(chat_id,user_id)`, индексы `chat_members(user_id,chat_id)`, `chats(created_at,id)` и `messages(chat_id,created_at,id)`. Создание чата с участниками и начальной версией — одна транзакция. Вставка сообщения и увеличение версии — одна транзакция. SQL параметризован; пул ограничен 10 соединениями, ожидание пула и подключение ограничены 3 с, `statement_timeout=3000` мс, `lock_timeout=2000` мс. Пароль БД передаётся отдельным параметром и безопасно экранируется `make_conninfo`, включая спецсимволы. Подтверждение COMMIT опирается на PostgreSQL с обычным `synchronous_commit=on` и сохранность volume/хоста. -## Эксперименты и отказы +При GET истории приложение проверяет сессию, членство и текущую версию чата в PostgreSQL. Ключ Redis включает chat ID, версию, limit и cursor. Запись подтверждается после COMMIT вместе с новой версией; следующий GET не использует кэш старой версии. При отказе Redis история читается из PostgreSQL, при отказе PostgreSQL обязательный путь и `/health/ready` возвращают 503. `/health/live` проверяет только процесс. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Проверки уровней 3 и 4 -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Критерий | Команда | Результат | +| --- | --- | --- | +| Публичный контракт | `make test` | PASS: Basic/Bearer, чаты, история, пагинация, ошибки, logout | +| Полный рестарт | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: старый токен и сообщение после остановки/старта API, PostgreSQL и Redis без удаления volume | +| Гонка username | `python3 tests/level4.py` | 16 конкурентных запросов: один 201, пятнадцать 409 | +| Rollback | та же команда | Тестовый trigger вызывает ошибку при вставке участника; чат отсутствует после отката | +| Отказы и expiry | та же команда | Redis down: чтение/запись работают; PostgreSQL down: 503; TTL 2 с: 401; анонимный доступ не принят | +| Миграции | полный рестарт выше | Повторный запуск сохранил данные и не применил миграцию повторно | +| Индекс истории | `python3 scripts/cache_experiment.py` на 10 000 строках | `Index Scan using messages_history_idx`, 101 строка, 6 buffer hits, 0.083 мс; полный план в `evidence/history_explain.txt` | +| Линт | `make lint` | PASS: синтаксис, пробелы, неиспользуемые импорты в изменённом Python-коде | -## Ограничения и следующие шаги +План получен запросом `SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id='' ORDER BY created_at,id LIMIT 101`. Следующая страница использует `(created_at,id)>(cursor_at,cursor_id)` и тот же индекс; полный scan истории для каждой страницы не требуется. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Трек 1: гипотеза и эксперимент + +**Гипотеза:** 32 одновременных запроса холодной страницы без защиты дадут cache stampede; Redis `SET NX PX` и ожидание результата сократят число SQL-запросов истории, а версия PostgreSQL обеспечит read-after-write. + +`python3 scripts/cache_experiment.py` создаёт синтетического пользователя, чат и 10 000 сообщений одной транзакцией. Затем отправляет 32 одновременных GET первой страницы (`limit=100`) в трёх режимах: кэш выключен, кэш без lock, кэш с lock. Перед каждым холодным прогоном очищает Redis и пересоздаёт API; затем повторяет 32 GET на прогретом кэше. Для воспроизводимой конкуренции во всех режимах эксперимент задаёт искусственную задержку cache miss 150 мс; в обычном запуске она равна нулю. `history_sql` считает именно SELECT страницы, без проверок сессии, членства и версии. Отдельная проба кэширует пустую страницу, записывает сообщение, сразу читает его, перезапускает Redis и читает снова. Эндпоинт счётчиков работает только при локальном `ENABLE_EXPERIMENT_METRICS=1` и доступен из контейнера. + +Три независимых запуска сохранены в `evidence/cache_experiment_run1.json` … `run3.json`: + +| Режим | Холодный SELECT истории / 32 GET | Прогретый SELECT / 32 GET | Холодный p95, мс (3 повтора) | Прогретый p95, мс (3 повтора) | +| --- | ---: | ---: | --- | --- | +| Без кэша | 32 | 32 | 666.36 / 690.73 / 686.28 | 680.33 / 684.34 / 665.88 | +| Кэш без lock | 10 | 0 | 254.70 / 253.97 / 243.40 | 103.82 / 102.69 / 104.81 | +| Кэш с lock | 1 | 0 | 230.95 / 248.14 / 239.43 | 105.44 / 102.45 / 107.05 | + +Во всех трёх запусках `read_after_write=true`, `cache_restart_recovered=true`, HTTP-ошибок не было. При одинаковом включённом кэше lock сократил дублирующиеся SQL-запросы холодной страницы с 10 до 1. Проверка версии по-прежнему требует SQL при каждом GET. Если SELECT длится дольше 500 мс ожидания, конкурентные клиенты могут снова выполнить запрос: lock с TTL 2 с не является распределённой транзакцией. Уже начавшееся чтение при конкурентной вставке может видеть прежний снимок; чтение после подтверждённой записи получает новую версию. Snapshot между HTTP-запросами не обещается. + +Отдельная 30-секундная нагрузка `python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json`: 16 331 успешный POST, 0 ошибок, 543.96 RPS, p50/p95/p99 = 7.088/9.878/12.638 мс. Один срез `docker stats --no-stream` во время нагрузки: API 128.04% CPU / 58.87 MiB; PostgreSQL 25.94% / 32.18 MiB; Redis 0.39% / 9.32 MiB. Closed-loop клиент имеет coordinated omission; один срез CPU/RAM не показывает пики. Все задержки локальные, без удалённой сети и лимитов CPU. + +## Ограничения + +HTTP допустим только на локальном изолированном стенде с синтетическими данными. Истёкшие сессии не очищаются фоново, но доступ к ним отклоняется в SQL. Старые версии страниц кэша удаляются по TTL. Результаты зависят от распределения размеров чатов, диска, ресурсов Docker и клиента; измерения не заявляют производительность production-системы. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..53ea2692 --- /dev/null +++ b/app/main.py @@ -0,0 +1,475 @@ +"""Lab 2 messenger: PostgreSQL authority, optional Redis page cache.""" + +import asyncio +import base64 +import binascii +import hashlib +import json +import os +import re +import secrets +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone + +import psycopg +import redis +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from psycopg.conninfo import make_conninfo +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException + + +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger") +if "DATABASE_PASSWORD" in os.environ: + DATABASE_URL = make_conninfo( + host=os.environ.get("DATABASE_HOST", "postgres"), + dbname=os.environ.get("DATABASE_NAME", "messenger"), + user=os.environ.get("DATABASE_USER", "messenger"), + password=os.environ["DATABASE_PASSWORD"], + ) +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") +SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) +MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384")) +POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10")) +CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1" +CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1" +CACHE_MISS_DELAY_MS = min(1000, max(0, int(os.environ.get("CACHE_MISS_DELAY_MS", "0")))) +pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=POOL_SIZE, timeout=3, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 3, "options": "-c statement_timeout=3000 -c lock_timeout=2000"}) +cache = redis.Redis.from_url(REDIS_URL, socket_connect_timeout=0.2, socket_timeout=0.2) +metrics = {"history_sql": 0, "history_cache_hit": 0, "history_lock_wait": 0} +metrics_lock = threading.Lock() + + +class ApiError(Exception): + def __init__(self, status, code, message, challenge=None): + self.status, self.code, self.message, self.challenge = status, code, message, challenge + + +def invalid(message="Некорректный запрос"): + raise ApiError(400, "invalid_request", message) + + +def unavailable(): + raise ApiError(503, "unavailable", "Хранилище временно недоступно") + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(scope, date, identifier): + raw = json.dumps([scope, date, identifier], separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def decode_cursor(scope, value): + if value is None: + return None + try: + raw = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + parts = json.loads(raw) + if (not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope + or not isinstance(parts[1], str) or not isinstance(parts[2], str)): + invalid("Некорректный cursor") + date = datetime.fromisoformat(parts[1].replace("Z", "+00:00")) + if date.tzinfo is None: + invalid("Некорректный cursor") + return date, str(uuid.UUID(parts[2])) + except (ValueError, TypeError, binascii.Error, IndexError): + invalid("Некорректный cursor") + + +def identifier(value): + try: + result = str(uuid.UUID(value)) + if result != value.lower() or len(value) != 36: + invalid("Некорректный UUID") + return result + except (ValueError, TypeError, AttributeError): + invalid("Некорректный UUID") + + +def text_field(value, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum: + invalid("Некорректное текстовое поле") + return value + + +def exact_body(value, fields): + if not isinstance(value, dict) or set(value) != set(fields): + invalid("Некорректные поля JSON") + + +async def body(request, fields): + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + declared = request.headers.get("content-length") + if declared is not None and declared.isdecimal() and int(declared) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + chunks = bytearray() + async for chunk in request.stream(): + chunks.extend(chunk) + if len(chunks) > MAX_BODY: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + try: + value = json.loads(chunks.decode("utf-8")) + except (UnicodeError, ValueError): + invalid("Некорректный JSON") + exact_body(value, fields) + return value + + +def user(row): + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])} + + +def message(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": stamp(row["created_at"])} + + +def chat(connection, row): + members = connection.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall() + return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(item["user_id"]) for item in members], "created_at": stamp(row["created_at"])} + + +def password_hash(password): + salt = os.urandom(16) + derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1) + return "scrypt:16384:8:1:" + salt.hex() + ":" + derived.hex() + + +def password_ok(password, stored): + try: + _, n, r, p, salt, expected = stored.split(":") + actual = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=int(n), r=int(r), p=int(p)) + return secrets.compare_digest(actual, bytes.fromhex(expected)) + except (ValueError, TypeError): + return False + + +def connection(): + try: + return pool.connection() + except psycopg.Error: + unavailable() + + +def actor(request): + auth = request.headers.get("authorization", "") + if not auth.startswith("Bearer ") or not auth[7:]: + raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"') + digest = hashlib.sha256(auth[7:].encode()).digest() + with connection() as db: + row = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()", (digest,)).fetchone() + if row is None: + raise ApiError(401, "unauthorized", "Требуется сессия", 'Bearer realm="messenger"') + return row["user_id"] + + +def member(db, uid, cid): + row = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s", (cid, uid)).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return row + + +def pagination(request): + raw = request.query_params.get("limit", "50") + if len(raw) > 20 or not re.fullmatch(r"[0-9]+", raw) or not 1 <= int(raw) <= 100: + invalid("Некорректный limit") + return int(raw) + + +def bump(key): + with metrics_lock: + metrics[key] += 1 + + +def cache_get(key): + try: + raw = cache.get(key) + if raw is not None: + bump("history_cache_hit") + return json.loads(raw) + except (redis.RedisError, ValueError): + pass + return None + + +def cache_set(key, data): + try: + cache.set(key, json.dumps(data), ex=30) + except redis.RedisError: + pass + + +def history(db, cid, limit, cursor, version): + key = f"history:{cid}:{version}:{limit}:{cursor or '-'}" + hit = cache_get(key) if CACHE_ENABLED else None + if hit is not None: + return hit + lock_key = "lock:" + key + token = secrets.token_hex(16) + locked = False + if CACHE_ENABLED and CACHE_LOCK_ENABLED: + try: + locked = bool(cache.set(lock_key, token, nx=True, px=2000)) + if not locked: + bump("history_lock_wait") + for _ in range(20): + time.sleep(0.025) + hit = cache_get(key) + if hit is not None: + return hit + locked = bool(cache.set(lock_key, token, nx=True, px=2000)) + except redis.RedisError: + pass + if CACHE_MISS_DELAY_MS: + time.sleep(CACHE_MISS_DELAY_MS / 1000) + if cursor is None: + rows = db.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s", (cid, limit + 1)).fetchall() + else: + rows = db.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s", (cid, cursor[0], cursor[1], limit + 1)).fetchall() + bump("history_sql") + result = {"items": [message(row) for row in rows[:limit]], "next_cursor": None} + if len(rows) > limit: + last = rows[limit - 1] + result["next_cursor"] = encode_cursor(str(cid), stamp(last["created_at"]), str(last["id"])) + if CACHE_ENABLED: + cache_set(key, result) + if locked: + try: + cache.eval("if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('del',KEYS[1]) end", 1, lock_key, token) + except redis.RedisError: + pass + return result + + +def migrate(): + from pathlib import Path + files = sorted(Path(__file__).resolve().parent.parent.joinpath("migrations").glob("*.sql")) + with pool.connection() as db: + with db.transaction(): + db.execute("SELECT pg_advisory_xact_lock(2172002)") + db.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)") + for file in files: + if db.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (file.name,)).fetchone() is None: + db.execute(file.read_text()) + db.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (file.name,)) + + +@asynccontextmanager +async def lifespan(_app): + pool.open() + pool.wait(timeout=20) + migrate() + yield + pool.close() + + +app = FastAPI(lifespan=lifespan) + + +@app.exception_handler(StarletteHTTPException) +async def http_error(request, error): + request_id = request.state.request_id + code = "not_found" if error.status_code == 404 else "invalid_request" + return JSONResponse({"error": {"code": code, "message": "Объект не найден" if error.status_code == 404 else "Некорректный запрос", "request_id": request_id}}, status_code=error.status_code) + + +@app.middleware("http") +async def envelope(request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except ApiError as exc: + response = JSONResponse({"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}, status_code=exc.status) + if exc.challenge: + response.headers["WWW-Authenticate"] = exc.challenge + except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable, PoolTimeout): + response = JSONResponse({"error": {"code": "unavailable", "message": "Хранилище временно недоступно", "request_id": request_id}}, status_code=503) + except Exception: + response = JSONResponse({"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}, status_code=500) + response.headers["X-Request-Id"] = request_id + return response + + +@app.get("/health/live") +def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def ready(): + with connection() as db: + db.execute("SELECT 1") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await body(request, ("username", "display_name", "password")) + name = data["username"] + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", name): + invalid("Некорректный username") + display = text_field(data["display_name"], 100) + password = data["password"] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + invalid("Некорректный пароль") + try: + return await asyncio.to_thread(insert_user, name, display, password) + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Username уже занят") from None + + +def insert_user(name, display, password): + digest = password_hash(password) + with connection() as db: + row = db.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), name, display, digest)).fetchone() + return user(row) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + async for chunk in request.stream(): + if chunk: + invalid("Тело запроса не допускается") + auth = request.headers.get("authorization", "") + try: + if not auth.startswith("Basic "): + raise ValueError() + decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8") + name, password = decoded.split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None + return await asyncio.to_thread(create_session, name, password) + + +def create_session(name, password): + with connection() as db: + row = db.execute("SELECT * FROM users WHERE username=%s", (name,)).fetchone() + if row is None or not password_ok(password, row["password_hash"]): + raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') + token = secrets.token_urlsafe(32) + expiry = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL) + db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (hashlib.sha256(token.encode()).digest(), row["id"], expiry)) + return {"token": token, "token_type": "Bearer", "expires_at": stamp(expiry), "user": user(row)} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + actor(request) + token = request.headers["authorization"][7:] + with connection() as db: + db.execute("DELETE FROM sessions WHERE token_hash=%s", (hashlib.sha256(token.encode()).digest(),)) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def me(request: Request): + uid = actor(request) + with connection() as db: + row = db.execute("SELECT * FROM users WHERE id=%s", (uid,)).fetchone() + return user(row) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + uid = actor(request) + data = await body(request, ("title", "member_ids")) + title = text_field(data["title"], 100) + raw = data["member_ids"] + if not isinstance(raw, list) or not 1 <= len(raw) <= 100: + invalid("Некорректный список участников") + members = [identifier(item) for item in raw] + if len(set(members)) != len(members) or str(uid) not in members: + invalid("Некорректный список участников") + return await asyncio.to_thread(insert_chat, uid, title, members) + + +def insert_chat(uid, title, members): + with connection() as db: + with db.transaction(): + count = db.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (members,)).fetchone()["n"] + if count != len(members): + invalid("Неизвестный участник") + row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), title)).fetchone() + for member_id in members: + db.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (row["id"], member_id)) + db.execute("INSERT INTO chat_versions(chat_id) VALUES (%s)", (row["id"],)) + return chat(db, row) + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + uid = actor(request) + limit = pagination(request) + cursor = decode_cursor("chats:" + str(uid), request.query_params.get("cursor")) + with connection() as db: + if cursor is None: + rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id LIMIT %s", (uid, limit + 1)).fetchall() + else: + valid = db.execute("SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND c.created_at=%s AND c.id=%s", (uid, cursor[0], cursor[1])).fetchone() + if valid is None: + invalid("Некорректный cursor") + rows = db.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s) ORDER BY c.created_at,c.id LIMIT %s", (uid, cursor[0], cursor[1], limit + 1)).fetchall() + items = [chat(db, row) for row in rows[:limit]] + next_cursor = encode_cursor("chats:" + str(uid), stamp(rows[limit - 1]["created_at"]), str(rows[limit - 1]["id"])) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + with connection() as db: + return chat(db, member(db, uid, cid)) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + data = await body(request, ("text",)) + content = text_field(data["text"], 4000) + return await asyncio.to_thread(insert_message, uid, cid, content) + + +def insert_message(uid, cid, content): + with connection() as db: + with db.transaction(): + member(db, uid, cid) + row = db.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), cid, uid, content)).fetchone() + db.execute("UPDATE chat_versions SET version=version+1 WHERE chat_id=%s", (cid,)) + return message(row) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + uid = actor(request) + cid = identifier(chat_id) + limit = pagination(request) + cursor = decode_cursor(cid, request.query_params.get("cursor")) + with connection() as db: + member(db, uid, cid) + if cursor is not None: + valid = db.execute("SELECT 1 FROM messages WHERE chat_id=%s AND created_at=%s AND id=%s", (cid, cursor[0], cursor[1])).fetchone() + if valid is None: + invalid("Некорректный cursor") + version = db.execute("SELECT version FROM chat_versions WHERE chat_id=%s", (cid,)).fetchone()["version"] + return history(db, cid, limit, cursor, version) + + +@app.get("/internal/cache-metrics") +def cache_metrics(request: Request): + if not os.environ.get("ENABLE_EXPERIMENT_METRICS") or request.client.host not in ("127.0.0.1", "::1"): + raise ApiError(404, "not_found", "Объект не найден") + with metrics_lock: + return dict(metrics) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..3fa757af --- /dev/null +++ b/compose.yaml @@ -0,0 +1,52 @@ +services: + postgres: + image: postgres:16.8-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 2s + retries: 20 + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 20 + api: + build: . + environment: + DATABASE_HOST: postgres + DATABASE_NAME: messenger + DATABASE_USER: messenger + DATABASE_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_SIZE: ${DB_POOL_SIZE:-10} + CACHE_ENABLED: ${CACHE_ENABLED:-1} + CACHE_LOCK_ENABLED: ${CACHE_LOCK_ENABLED:-1} + CACHE_MISS_DELAY_MS: ${CACHE_MISS_DELAY_MS:-0} + ENABLE_EXPERIMENT_METRICS: ${ENABLE_EXPERIMENT_METRICS:-} + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 3s + timeout: 3s + start_period: 5s + retries: 20 +volumes: + postgres_data: diff --git a/evidence/cache_experiment_run1.json b/evidence/cache_experiment_run1.json new file mode 100644 index 00000000..03599493 --- /dev/null +++ b/evidence/cache_experiment_run1.json @@ -0,0 +1,63 @@ +{ + "seed_messages": 10000, + "concurrency": 32, + "cache_miss_delay_ms": 150, + "modes": { + "cache_off": { + "cold": { + "requests": 32, + "p50_ms": 361.21, + "p95_ms": 666.36, + "p99_ms": 666.37 + }, + "warm": { + "requests": 32, + "p50_ms": 351.69, + "p95_ms": 680.33, + "p99_ms": 680.7 + }, + "cold_history_sql": 32, + "warm_history_sql": 32, + "cold_cache_hits": 0, + "cold_lock_waits": 0 + }, + "cache_unlocked": { + "cold": { + "requests": 32, + "p50_ms": 237.13, + "p95_ms": 254.7, + "p99_ms": 254.95 + }, + "warm": { + "requests": 32, + "p50_ms": 75.55, + "p95_ms": 103.82, + "p99_ms": 104.44 + }, + "cold_history_sql": 10, + "warm_history_sql": 0, + "cold_cache_hits": 22, + "cold_lock_waits": 0 + }, + "cache_locked": { + "cold": { + "requests": 32, + "p50_ms": 219.87, + "p95_ms": 230.95, + "p99_ms": 232.52 + }, + "warm": { + "requests": 32, + "p50_ms": 85.9, + "p95_ms": 105.44, + "p99_ms": 105.49 + }, + "cold_history_sql": 1, + "warm_history_sql": 0, + "cold_cache_hits": 31, + "cold_lock_waits": 9 + } + }, + "read_after_write": true, + "cache_restart_recovered": true +} diff --git a/evidence/cache_experiment_run2.json b/evidence/cache_experiment_run2.json new file mode 100644 index 00000000..5b031401 --- /dev/null +++ b/evidence/cache_experiment_run2.json @@ -0,0 +1,63 @@ +{ + "seed_messages": 10000, + "concurrency": 32, + "cache_miss_delay_ms": 150, + "modes": { + "cache_off": { + "cold": { + "requests": 32, + "p50_ms": 373.74, + "p95_ms": 690.73, + "p99_ms": 693.48 + }, + "warm": { + "requests": 32, + "p50_ms": 368.38, + "p95_ms": 684.34, + "p99_ms": 684.77 + }, + "cold_history_sql": 32, + "warm_history_sql": 32, + "cold_cache_hits": 0, + "cold_lock_waits": 0 + }, + "cache_unlocked": { + "cold": { + "requests": 32, + "p50_ms": 248.52, + "p95_ms": 253.97, + "p99_ms": 254.91 + }, + "warm": { + "requests": 32, + "p50_ms": 78.15, + "p95_ms": 102.69, + "p99_ms": 103.55 + }, + "cold_history_sql": 10, + "warm_history_sql": 0, + "cold_cache_hits": 22, + "cold_lock_waits": 0 + }, + "cache_locked": { + "cold": { + "requests": 32, + "p50_ms": 238.68, + "p95_ms": 248.14, + "p99_ms": 248.63 + }, + "warm": { + "requests": 32, + "p50_ms": 78.41, + "p95_ms": 102.45, + "p99_ms": 102.47 + }, + "cold_history_sql": 1, + "warm_history_sql": 0, + "cold_cache_hits": 31, + "cold_lock_waits": 9 + } + }, + "read_after_write": true, + "cache_restart_recovered": true +} diff --git a/evidence/cache_experiment_run3.json b/evidence/cache_experiment_run3.json new file mode 100644 index 00000000..f48d9e83 --- /dev/null +++ b/evidence/cache_experiment_run3.json @@ -0,0 +1,63 @@ +{ + "seed_messages": 10000, + "concurrency": 32, + "cache_miss_delay_ms": 150, + "modes": { + "cache_off": { + "cold": { + "requests": 32, + "p50_ms": 377.37, + "p95_ms": 686.28, + "p99_ms": 686.28 + }, + "warm": { + "requests": 32, + "p50_ms": 358.32, + "p95_ms": 665.88, + "p99_ms": 666.82 + }, + "cold_history_sql": 32, + "warm_history_sql": 32, + "cold_cache_hits": 0, + "cold_lock_waits": 0 + }, + "cache_unlocked": { + "cold": { + "requests": 32, + "p50_ms": 219.65, + "p95_ms": 243.4, + "p99_ms": 243.53 + }, + "warm": { + "requests": 32, + "p50_ms": 81.49, + "p95_ms": 104.81, + "p99_ms": 109.72 + }, + "cold_history_sql": 10, + "warm_history_sql": 0, + "cold_cache_hits": 22, + "cold_lock_waits": 0 + }, + "cache_locked": { + "cold": { + "requests": 32, + "p50_ms": 229.83, + "p95_ms": 239.43, + "p99_ms": 239.86 + }, + "warm": { + "requests": 32, + "p50_ms": 79.86, + "p95_ms": 107.05, + "p99_ms": 109.8 + }, + "cold_history_sql": 1, + "warm_history_sql": 0, + "cold_cache_hits": 31, + "cold_lock_waits": 9 + } + }, + "read_after_write": true, + "cache_restart_recovered": true +} diff --git a/evidence/history_explain.txt b/evidence/history_explain.txt new file mode 100644 index 00000000..6b58d946 --- /dev/null +++ b/evidence/history_explain.txt @@ -0,0 +1,15 @@ +EXPLAIN (ANALYZE, BUFFERS) SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id='b882362b-f5e0-4a9c-84f9-1dbe79d7e388' ORDER BY created_at,id LIMIT 101; + + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.42..35.63 rows=101 width=66) (actual time=0.021..0.043 rows=101 loops=1) + Buffers: shared hit=6 + -> Index Scan using messages_history_idx on messages (cost=0.42..3483.99 rows=9993 width=66) (actual time=0.020..0.034 rows=101 loops=1) + Index Cond: (chat_id = 'b882362b-f5e0-4a9c-84f9-1dbe79d7e388'::uuid) + Buffers: shared hit=6 + Planning: + Buffers: shared hit=107 + Planning Time: 0.434 ms + Execution Time: 0.083 ms +(9 rows) + diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..d3bbacd1 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 2, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.022, + "concurrency": 4, + "successful_requests": 16331, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 543.96, + "successful_latency_ms": { + "p50": 7.088, + "p95": 9.878, + "p99": 12.638 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..2cdfbf05 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,38 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$') +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); +CREATE INDEX chats_order_idx ON chats (created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +CREATE TABLE chat_versions ( + chat_id uuid PRIMARY KEY REFERENCES chats(id), + version bigint NOT NULL DEFAULT 0 +); +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..1dc75f74 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +psycopg[binary,pool]==3.2.6 +redis==5.2.1 diff --git a/scripts/cache_experiment.py b/scripts/cache_experiment.py new file mode 100644 index 00000000..abbfc35d --- /dev/null +++ b/scripts/cache_experiment.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Reproducible isolated Compose experiment for track 1.""" + +import base64 +import concurrent.futures +import json +import math +import os +import statistics +import subprocess +import time +import urllib.request +import uuid +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://localhost:8080") + + +def compose(*args, env=None): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, env=env, check=True, capture_output=True, text=True).stdout + + +def api(method, path, token=None, data=None, extra=None): + headers = {"Accept": "application/json"} + if token: + headers["Authorization"] = "Bearer " + token + if data is not None: + headers["Content-Type"] = "application/json" + headers.update(extra or {}) + raw = json.dumps(data).encode() if data is not None else None + request = urllib.request.Request(BASE + path, method=method, headers=headers, data=raw) + start = time.perf_counter() + with urllib.request.urlopen(request, timeout=10) as response: + payload = json.loads(response.read()) + return payload, (time.perf_counter() - start) * 1000 + + +def metrics(): + code = "import json,urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/internal/cache-metrics').read().decode())" + return json.loads(compose("exec", "-T", "api", "python", "-c", code)) + + +def run_batch(path, token, count=32): + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as workers: + results = list(workers.map(lambda _: api("GET", path, token), range(count))) + assert all(item[0] == results[0][0] for item in results) + times = sorted(item[1] for item in results) + return {"requests": count, "p50_ms": round(statistics.median(times), 2), "p95_ms": round(times[math.ceil(count * 0.95) - 1], 2), "p99_ms": round(times[-1], 2)} + + +def restart_api(cache_enabled, lock_enabled): + env = dict(os.environ, CACHE_ENABLED="1" if cache_enabled else "0", + CACHE_LOCK_ENABLED="1" if lock_enabled else "0", + CACHE_MISS_DELAY_MS="150", ENABLE_EXPERIMENT_METRICS="1") + compose("up", "-d", "--force-recreate", "--wait", "api", env=env) + + +def main(): + username = "experiment_" + uuid.uuid4().hex[:12] + password = "synthetic-" + uuid.uuid4().hex + account, _ = api("POST", "/api/v1/users", data={"username": username, "display_name": "Experiment", "password": password}) + basic = base64.b64encode((username + ":" + password).encode()).decode() + session, _ = api("POST", "/api/v1/auth/sessions", extra={"Authorization": "Basic " + basic}) + token = session["token"] + chat, _ = api("POST", "/api/v1/chats", token, {"title": "Cache experiment", "member_ids": [account["id"]]}) + chat_id = chat["id"] + # Synthetic seed in one transaction; the same indexed data serves both modes. + sql = ("BEGIN; INSERT INTO messages(id,chat_id,sender_id,text) " + f"SELECT gen_random_uuid(),'{chat_id}'::uuid,'{account['id']}'::uuid,'seed-'||g " + "FROM generate_series(1,10000) g; " + f"UPDATE chat_versions SET version=version+1 WHERE chat_id='{chat_id}'; COMMIT;") + compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", sql) + explain_sql = ("EXPLAIN (ANALYZE, BUFFERS) SELECT id,chat_id,sender_id,text,created_at " + f"FROM messages WHERE chat_id='{chat_id}' ORDER BY created_at,id LIMIT 101;") + compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", "ANALYZE messages;") + plan = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", explain_sql) + (ROOT / "evidence" / "history_explain.txt").write_text(explain_sql + "\n\n" + plan) + path = f"/api/v1/chats/{chat_id}/messages?limit=100" + result = {"seed_messages": 10000, "concurrency": 32, "cache_miss_delay_ms": 150, "modes": {}} + for label, cache_enabled, lock_enabled in (("cache_off", False, False), ("cache_unlocked", True, False), ("cache_locked", True, True)): + restart_api(cache_enabled, lock_enabled) + compose("exec", "-T", "redis", "redis-cli", "FLUSHDB") + before = metrics() + cold = run_batch(path, token) + middle = metrics() + warm = run_batch(path, token) + after = metrics() + result["modes"][label] = { + "cold": cold, "warm": warm, + "cold_history_sql": middle["history_sql"] - before["history_sql"], + "warm_history_sql": after["history_sql"] - middle["history_sql"], + "cold_cache_hits": middle["history_cache_hit"] - before["history_cache_hit"], + "cold_lock_waits": middle["history_lock_wait"] - before["history_lock_wait"], + } + # A write changes the database version in the same transaction as the message. + fresh, _ = api("POST", "/api/v1/chats", token, {"title": "Consistency probe", "member_ids": [account["id"]]}) + fresh_path = f"/api/v1/chats/{fresh['id']}/messages?limit=100" + api("GET", fresh_path, token) # cache an empty page + new, _ = api("POST", f"/api/v1/chats/{fresh['id']}/messages", token, {"text": "read after write"}) + after = api("GET", fresh_path, token)[0] + result["read_after_write"] = any(item["id"] == new["id"] for item in after["items"]) + # The first page is restored from PostgreSQL after the disposable cache restarts. + compose("restart", "redis") + recovered = api("GET", fresh_path, token)[0] + result["cache_restart_recovered"] = any(item["id"] == new["id"] for item in recovered["items"]) + assert result["read_after_write"] and result["cache_restart_recovered"] + output = ROOT / "evidence" / os.environ.get("EVIDENCE_OUTPUT", "cache_experiment.json") + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print(output) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/lint.py b/scripts/lint.py new file mode 100644 index 00000000..5faea76e --- /dev/null +++ b/scripts/lint.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Offline lint for this small Python project: syntax, whitespace, unused imports.""" + +import ast +from pathlib import Path +import sys + + +ROOT = Path(__file__).resolve().parents[1] +FILES = [ROOT / "app/main.py", ROOT / "scripts/cache_experiment.py", ROOT / "scripts/lint.py", ROOT / "tests/level4.py"] + + +def check(path): + source = path.read_text() + issues = [] + try: + tree = ast.parse(source, filename=str(path)) + except SyntaxError as error: + return [f"{path}:{error.lineno}: {error.msg}"] + used = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports = [(item.asname or item.name.split(".")[0], node.lineno) for item in node.names] + elif isinstance(node, ast.ImportFrom): + imports = [(item.asname or item.name, node.lineno) for item in node.names] + else: + continue + for name, line in imports: + if name not in used: + issues.append(f"{path}:{line}: unused import {name}") + for number, line in enumerate(source.splitlines(), 1): + if line.rstrip() != line: + issues.append(f"{path}:{number}: trailing whitespace") + if "\t" in line: + issues.append(f"{path}:{number}: tab character") + return issues + + +def main(): + issues = [issue for path in FILES for issue in check(path)] + if issues: + print("\n".join(issues)) + return 1 + print(f"PASS: lint ({len(FILES)} Python files)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..3a767bc5 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop api redis postgres +docker compose --env-file .env up -d --wait postgres redis api diff --git a/tests/level4.py b/tests/level4.py new file mode 100644 index 00000000..aec8a7c4 --- /dev/null +++ b/tests/level4.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Extra integration checks for lab 2 level 4 on a disposable local stack.""" + +import base64 +import concurrent.futures +import json +import os +from pathlib import Path +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid + + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://localhost:8080") + + +def compose(*args, env=None): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, env=env, check=True, capture_output=True, text=True).stdout + + +def psql(sql): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A", "-c", sql) + + +def call(method, path, data=None, auth=None): + headers = {"Accept": "application/json"} + if data is not None: + headers["Content-Type"] = "application/json" + if auth: + headers["Authorization"] = auth + request = urllib.request.Request(BASE + path, method=method, headers=headers, data=json.dumps(data).encode() if data is not None else None) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read() or b"null") + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + + +def register(name, password): + return call("POST", "/api/v1/users", {"username": name, "display_name": "Synthetic", "password": password}) + + +def login(name, password): + basic = base64.b64encode((name + ":" + password).encode()).decode() + return call("POST", "/api/v1/auth/sessions", auth="Basic " + basic) + + +def main(): + name = "race_" + uuid.uuid4().hex[:12] + password = "synthetic-" + uuid.uuid4().hex + gate = threading.Barrier(16) + def competing_register(_): + gate.wait() + return register(name, password)[0] + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as workers: + statuses = list(workers.map(competing_register, range(16))) + assert statuses.count(201) == 1 and statuses.count(409) == 15, statuses + status, session = login(name, password) + assert status == 201 + basic = base64.b64encode((name + ":" + password).encode()).decode() + assert call("POST", "/api/v1/auth/sessions", {"unexpected": "body"}, "Basic " + basic)[0] == 400 + token = "Bearer " + session["token"] + user_id = session["user"]["id"] + status, chat = call("POST", "/api/v1/chats", {"title": "Failure test", "member_ids": [user_id]}, token) + assert status == 201 + probe = "rollback_" + uuid.uuid4().hex + psql("CREATE FUNCTION lab2_rollback_guard() RETURNS trigger LANGUAGE plpgsql AS $$ " + f"BEGIN IF EXISTS (SELECT 1 FROM chats WHERE id=NEW.chat_id AND title='{probe}') " + "THEN RAISE EXCEPTION 'rollback probe'; END IF; RETURN NEW; END $$;") + psql("CREATE TRIGGER lab2_rollback_guard BEFORE INSERT ON chat_members " + "FOR EACH ROW EXECUTE FUNCTION lab2_rollback_guard();") + try: + assert call("POST", "/api/v1/chats", {"title": probe, "member_ids": [user_id]}, token)[0] == 500 + assert psql(f"SELECT count(*) FROM chats WHERE title='{probe}'").strip() == "0" + finally: + psql("DROP TRIGGER lab2_rollback_guard ON chat_members; DROP FUNCTION lab2_rollback_guard();") + path = "/api/v1/chats/" + chat["id"] + "/messages" + assert call("POST", path, {"text": "before outage"}, token)[0] == 201 + malformed = base64.urlsafe_b64encode(json.dumps([chat["id"], 17, user_id]).encode()).decode().rstrip("=") + assert call("GET", path + "?cursor=" + malformed, auth=token)[0] == 400 + assert call("GET", path + "?limit=" + "9" * 5000, auth=token)[0] == 400 + missing_status, missing_body = call("GET", "/api/v1/missing", auth=token) + assert missing_status == 404 and missing_body["error"]["code"] == "not_found" + compose("stop", "redis") + try: + assert call("GET", path, auth=token)[0] == 200 + assert call("POST", path, {"text": "during cache outage"}, token)[0] == 201 + assert call("GET", path, auth=token)[1]["items"][-1]["text"] == "during cache outage" + assert call("GET", "/api/v1/users/me", auth="Bearer invalid")[0] == 401 + finally: + compose("start", "redis") + compose("stop", "postgres") + try: + assert call("GET", "/health/ready")[0] == 503 + assert call("GET", "/api/v1/users/me", auth=token)[0] == 503 + finally: + compose("start", "postgres") + for _ in range(40): + if call("GET", "/health/ready")[0] == 200: + break + time.sleep(0.5) + else: + raise AssertionError("PostgreSQL did not recover") + # Expiry is checked against PostgreSQL time, without a Redis dependency. + env = dict(os.environ, SESSION_TTL_SECONDS="2") + compose("up", "-d", "--force-recreate", "--wait", "api", env=env) + try: + short = login(name, password)[1]["token"] + time.sleep(3) + assert call("GET", "/api/v1/users/me", auth="Bearer " + short)[0] == 401 + finally: + compose("up", "-d", "--force-recreate", "--wait", "api") + print("PASS: username race, chat rollback, Redis fallback, PostgreSQL failure, session expiry") + + +if __name__ == "__main__": + main() -- 2.54.0 From 4cba24863a53070a91989f24ca3b31c2d947eb74 Mon Sep 17 00:00:00 2001 From: "yurtaev.va" Date: Sat, 24 Oct 2026 13:20:09 +0300 Subject: [PATCH 2/2] review fixes --- .env.example | 1 + REPORT.md | 4 ++++ app/main.py | 8 ++++++-- compose.yaml | 1 + tests/level4.py | 12 ++++++++++-- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index f55e7768..3c0cce2b 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,7 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 +MAX_HTTP_BODY_BYTES=65536 # Set a local synthetic password in .env; never commit it. POSTGRES_PASSWORD=replace-me-with-a-local-password DB_POOL_SIZE=10 diff --git a/REPORT.md b/REPORT.md index d2a95b07..d9515ac7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -17,6 +17,9 @@ python3 tests/level4.py ``` API открыт только на `127.0.0.1:8080`. При первом старте миграции `migrations/*.sql` применяются под транзакционной advisory lock; повторный старт их пропускает. Остановка: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. `.env` и volume исключены из Git. +Если порт 8080 занят, можно задать `APP_PORT=8082` для Compose и `BASE_URL=http://localhost:8082` для тестов; финальные регрессии и persistence были повторены так из-за другого локального стенда на 8080. + +Лимит HTTP JSON-тела `MAX_HTTP_BODY_BYTES` по умолчанию 65 536 байт и не может быть меньше: это позволяет передать 4000 Unicode-символов даже при ASCII-экранировании `json.dumps`. Длина текста и пароля проверяется уже после декодирования JSON; пароль из одних пробельных символов отклоняется. ## Архитектура @@ -35,6 +38,7 @@ API открыт только на `127.0.0.1:8080`. При первом ста | Гонка username | `python3 tests/level4.py` | 16 конкурентных запросов: один 201, пятнадцать 409 | | Rollback | та же команда | Тестовый trigger вызывает ошибку при вставке участника; чат отсутствует после отката | | Отказы и expiry | та же команда | Redis down: чтение/запись работают; PostgreSQL down: 503; TTL 2 с: 401; анонимный доступ не принят | +| Границы JSON и пароля | та же команда | 4000 emoji проходят с ASCII-экранированием (48 012 байт) и без него (16 012 байт); пароль из 12 пробелов получает 400 | | Миграции | полный рестарт выше | Повторный запуск сохранил данные и не применил миграцию повторно | | Индекс истории | `python3 scripts/cache_experiment.py` на 10 000 строках | `Index Scan using messages_history_idx`, 101 строка, 6 buffer hits, 0.083 мс; полный план в `evidence/history_explain.txt` | | Линт | `make lint` | PASS: синтаксис, пробелы, неиспользуемые импорты в изменённом Python-коде | diff --git a/app/main.py b/app/main.py index 53ea2692..430180c0 100644 --- a/app/main.py +++ b/app/main.py @@ -34,7 +34,9 @@ if "DATABASE_PASSWORD" in os.environ: ) REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") SESSION_TTL = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) -MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "16384")) +MAX_BODY = int(os.environ.get("MAX_HTTP_BODY_BYTES", "65536")) +if MAX_BODY < 65536: + raise ValueError("MAX_HTTP_BODY_BYTES must be at least 65536") POOL_SIZE = int(os.environ.get("DB_POOL_SIZE", "10")) CACHE_ENABLED = os.environ.get("CACHE_ENABLED", "1") == "1" CACHE_LOCK_ENABLED = os.environ.get("CACHE_LOCK_ENABLED", "1") == "1" @@ -321,7 +323,7 @@ async def create_user(request: Request): invalid("Некорректный username") display = text_field(data["display_name"], 100) password = data["password"] - if not isinstance(password, str) or not 12 <= len(password) <= 128: + if not isinstance(password, str) or not 12 <= len(password) <= 128 or not password.strip(): invalid("Некорректный пароль") try: return await asyncio.to_thread(insert_user, name, display, password) @@ -347,6 +349,8 @@ async def login(request: Request): raise ValueError() decoded = base64.b64decode(auth[6:], validate=True).decode("utf-8") name, password = decoded.split(":", 1) + if not re.fullmatch(r"[a-z0-9_]{3,32}", name) or not 12 <= len(password) <= 128 or not password.strip(): + raise ValueError() except (ValueError, UnicodeError, binascii.Error): raise ApiError(401, "unauthorized", "Неверные учётные данные", 'Basic realm="messenger", charset="UTF-8"') from None return await asyncio.to_thread(create_session, name, password) diff --git a/compose.yaml b/compose.yaml index 3fa757af..482cc789 100644 --- a/compose.yaml +++ b/compose.yaml @@ -29,6 +29,7 @@ services: DATABASE_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} REDIS_URL: redis://redis:6379/0 SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + MAX_HTTP_BODY_BYTES: ${MAX_HTTP_BODY_BYTES:-65536} DB_POOL_SIZE: ${DB_POOL_SIZE:-10} CACHE_ENABLED: ${CACHE_ENABLED:-1} CACHE_LOCK_ENABLED: ${CACHE_LOCK_ENABLED:-1} diff --git a/tests/level4.py b/tests/level4.py index aec8a7c4..8f80e36d 100644 --- a/tests/level4.py +++ b/tests/level4.py @@ -26,13 +26,13 @@ def psql(sql): return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A", "-c", sql) -def call(method, path, data=None, auth=None): +def call(method, path, data=None, auth=None, ensure_ascii=True): headers = {"Accept": "application/json"} if data is not None: headers["Content-Type"] = "application/json" if auth: headers["Authorization"] = auth - request = urllib.request.Request(BASE + path, method=method, headers=headers, data=json.dumps(data).encode() if data is not None else None) + request = urllib.request.Request(BASE + path, method=method, headers=headers, data=json.dumps(data, ensure_ascii=ensure_ascii).encode() if data is not None else None) try: with urllib.request.urlopen(request, timeout=5) as response: return response.status, json.loads(response.read() or b"null") @@ -59,6 +59,8 @@ def main(): with concurrent.futures.ThreadPoolExecutor(max_workers=16) as workers: statuses = list(workers.map(competing_register, range(16))) assert statuses.count(201) == 1 and statuses.count(409) == 15, statuses + spaces_name = "spaces_" + uuid.uuid4().hex[:12] + assert register(spaces_name, " " * 12)[0] == 400 status, session = login(name, password) assert status == 201 basic = base64.b64encode((name + ":" + password).encode()).decode() @@ -79,6 +81,12 @@ def main(): finally: psql("DROP TRIGGER lab2_rollback_guard ON chat_members; DROP FUNCTION lab2_rollback_guard();") path = "/api/v1/chats/" + chat["id"] + "/messages" + emoji_text = "😀" * 4000 + assert len(json.dumps({"text": emoji_text}).encode()) == 48012 + assert len(json.dumps({"text": emoji_text}, ensure_ascii=False).encode()) == 16012 + for ensure_ascii in (True, False): + status, sent = call("POST", path, {"text": emoji_text}, token, ensure_ascii=ensure_ascii) + assert status == 201 and sent["text"] == emoji_text assert call("POST", path, {"text": "before outage"}, token)[0] == 201 malformed = base64.urlsafe_b64encode(json.dumps([chat["id"], 17, user_id]).encode()).decode().rstrip("=") assert call("GET", path + "?cursor=" + malformed, auth=token)[0] == 400 -- 2.54.0