From f87929c0d712140c382780fcf11ffd0dd02c6ad0 Mon Sep 17 00:00:00 2001 From: Nik <121254636+nikikiki19@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:44:16 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=D0=9B=D0=A0=202=20=E2=80=94=20PostgreSQL,?= =?UTF-8?q?=20=D1=81=D0=B5=D1=81=D1=81=D0=B8=D0=B8=20=D0=B8=20Redis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 13 +- Dockerfile | 11 ++ Makefile | 5 +- REPORT.md | 73 +++++++--- app/__init__.py | 0 app/main.py | 262 +++++++++++++++++++++++++++++++++++ compose.yaml | 31 +++++ evidence/explain_history.txt | 13 ++ evidence/track_cache.json | 9 ++ evidence/track_cache.py | 30 ++++ migrations/001_init.sql | 19 +++ requirements.txt | 5 + scripts/explain_history.sql | 7 + scripts/migrate.py | 10 ++ scripts/restart.sh | 6 + tests/level4.py | 29 ++++ 16 files changed, 494 insertions(+), 29 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 compose.yaml create mode 100644 evidence/explain_history.txt create mode 100644 evidence/track_cache.json create mode 100755 evidence/track_cache.py create mode 100644 migrations/001_init.sql create mode 100644 requirements.txt create mode 100644 scripts/explain_history.sql create mode 100755 scripts/migrate.py create mode 100755 scripts/restart.sh create mode 100644 tests/level4.py diff --git a/.env.example b/.env.example index 714c8076..a680fad6 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,4 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-local SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c479385e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12.8-slim-bookworm +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app ./app +COPY migrations ./migrations +COPY scripts ./scripts +RUN useradd --create-home --uid 10001 appuser && chown -R appuser:appuser /app +USER appuser +EXPOSE 8080 +CMD ["sh", "-c", "python scripts/migrate.py && uvicorn app.main:app --host 0.0.0.0 --port 8080"] diff --git a/Makefile b/Makefile index 9ecb223e..604ef6c9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence level4 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,3 +24,6 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +level4: + $(PYTHON) tests/level4.py diff --git a/REPORT.md b/REPORT.md index e42f2b05..1d216570 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,74 @@ -# Отчёт — лабораторная 2 +# Отчёт по лабораторной работе № 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Студент: Борзов Никита +- Группа: БВТ2304 +- Целевая оценка: 5 +- Выбранный трек 5: трек 1, согласованность кэша +- Ветка/Commit/PR: заполняется после публикации + +В этой работе я продолжил REST-мессенджер из лабораторной 1. In-memory-хранилище заменено на PostgreSQL, добавлены пароль при регистрации, Basic-аутентификация только для входа и непрозрачные Bearer-сессии. Redis используется для TTL-сессий и кэша списка чатов. Сессии записываются в Redis с AOF и переживают перезапуск API и Redis-volume. + +Использованы контракт и публичные проверки из репозитория лабораторной 2, FastAPI, psycopg 3, PostgreSQL 16.4, Redis 7.4 и Argon2id из `argon2-cffi`. Код первой лабораторной и отчёт использованы как исходная предметная модель; при подготовке кода и проверки требований использовалась помощь ИИ, все команды и результаты проверены локально. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужен Docker Desktop с Compose v2: + +```sh +cp .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +docker compose ps +make check +make test BASE_URL=http://localhost:8080 +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080 +``` + +Миграция `scripts/migrate.py` запускается внутри API до Uvicorn и идемпотентна. Для остановки без удаления данных используется `docker compose stop`; намеренный сброс учебного стенда выполняется отдельно командой `docker compose down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент -> FastAPI/Uvicorn -> PostgreSQL + └── Redis (сессии TTL + кэш списка чатов) +``` + +PostgreSQL является источником истины для пользователей, чатов, членства и сообщений. Таблицы, внешние ключи, уникальность username и индекс истории описаны в `migrations/001_init.sql`. Создание чата и его участников выполняется одной транзакцией. Пул PostgreSQL ограничен десятью соединениями. + +Пароли хешируются Argon2id и никогда не возвращаются в User. Redis хранит `session:` с TTL из `SESSION_TTL_SECONDS`; токен случайный, длиной не менее 256 бит. Redis включён с AOF `everysec`, поэтому действующая сессия сохраняется при обычном перезапуске контейнера с volume. Кэш `chats:` живёт 30 секунд и удаляется при создании чата. После записи сообщение читается только из PostgreSQL, поэтому кэш не может скрыть подтверждённую запись. + +Курсор истории кодирует `(created_at, id)` и используется в параметризованном keyset-запросе. Все ответы получают `X-Request-Id` и `X-Instance-Id`, ошибки имеют единый контракт и не содержат SQL или stack trace. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт и структура | `make check` | `PASS: offline skeleton structure` | вывод команды | +| OpenAPI-схема | `make validate PYTHON=.venv/bin/python` | `PASS: full OpenAPI 3.1 and JSON Schema validation` | вывод команды | +| Публичный smoke | `make test` | `PASS: lab2 public API scenario` | вывод команды | +| PostgreSQL и миграции | `docker compose up -d --build` | API применяет `001_init.sql`, данные находятся в PostgreSQL | `migrations/001_init.sql` | +| Сохранность API/БД/сессии | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | подтверждённое сообщение и прежний Bearer-токен сохранены | `scripts/restart.sh` | +| Redis с TTL и volume | `docker compose ps`, `redis-cli TTL session:*` | Redis healthy, TTL уменьшается, AOF включён | `compose.yaml` | +| Индекс истории | `EXPLAIN (ANALYZE, BUFFERS)` на 10 000 синтетических сообщениях | `Index Scan using messages_history_idx`, 50 строк за 0.045 ms | `evidence/explain_history.txt` | +| Уровень 4: race/rollback/expiry | `python tests/level4.py` при TTL=2 | 1 регистрация получила 201, 7 гонок — 409; невалидный чат не появился; просроченная сессия — 401 | `tests/level4.py` | +| Трек 5: кэш | `python evidence/track_cache.py` | 20 конкурентных холодных чтений дали 1 SQL-запрос; после очистки и рестарта Redis — 1 запрос на восстановление; запись инвалидирует ключ | `evidence/track_cache.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Исследовательский трек: согласованность кэша + +Гипотеза: Redis-кэш уменьшит задержку повторного чтения списка чатов, а явная инвалидация после создания чата сохранит read-after-write. + +Методика: один синтетический пользователь на локальном Docker Compose, холодный `GET /api/v1/chats`, 30 последовательных чтений и 20 одновременных чтений после удаления ключа. Счётчик SQL-запросов включён только в локальном debug-режиме. В последнем прогоне получены cold 3.221 ms, warm p50 1.992 ms, warm p95 2.350 ms; 20 конкурентных запросов выполнили 1 SQL-чтение благодаря блокировке, после удаления ключа и рестарта Redis восстановление выполнило 1 SQL-чтение. После `POST /api/v1/chats` ключ пользователя удаляется; следующий GET заново читает PostgreSQL и возвращает новый чат. Ограничение измерения — один ноутбук и небольшой синтетический набор данных, поэтому результат показывает поведение защиты от stampede и направление задержки, а не универсальный RPS. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Ожидание | Наблюдение | Потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Перезапуск API и Redis/PostgreSQL через `scripts/restart.sh` | старый токен и подтверждённые данные сохраняются | публичный persistence-тест прошёл | 0 подтверждённых сообщений | +| Неверный Basic-пароль | одинаковый 401 и Basic challenge | проверено smoke | нет | +| Logout и повторный запрос | токен становится недействительным | DELETE возвращает 204, повтор — 401 | нет | +| Redis временно недоступен | readiness 503, анонимный запрос не принимается | получен HTTP 503 с `unavailable`, после запуска Redis — `{"status":"ok"}` | нет; восстановление зависит от Redis | +| Короткий TTL | `SESSION_TTL_SECONDS=2 ... tests/level4.py` | прежний токен получил 401 после истечения, гонка регистрации дала 1×201 и 7×409 | нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Сессии зависят от Redis: при удалении его volume активные токены отзываются, хотя PostgreSQL-данные остаются. В обязательном API нет удаления пользователей/чатов и управления участниками. Для лаборатории 3 потребуется вынести идемпотентность сообщений и общие сессии на несколько API-реплик, сохранив этот PostgreSQL-контур. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..04bc83f1 --- /dev/null +++ b/app/main.py @@ -0,0 +1,262 @@ +import base64, hashlib, json, os, secrets, threading, uuid +from datetime import datetime, timezone, timedelta +from pathlib import Path +from typing import Any + +import redis +from argon2 import PasswordHasher +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://messenger:messenger@postgres:5432/messenger") +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +INSTANCE_ID = os.getenv("INSTANCE_ID", uuid.uuid4().hex[:12]) +ph = PasswordHasher() +pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=int(os.getenv("DB_POOL_SIZE", "10")), timeout=5, kwargs={"row_factory": dict_row}, open=False) +cache = redis.Redis.from_url(REDIS_URL, decode_responses=True, socket_connect_timeout=1) +chat_cache_lock = threading.Lock() +cache_sql_reads = 0 +app = FastAPI(title="MTUSI Messenger Lab 2", version="1.2.0") + +def now(): return datetime.now(timezone.utc) +def iso(value): return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") +def uid(value): + try: return uuid.UUID(str(value)) + except (ValueError, TypeError): raise ValueError("invalid uuid") +def user_view(row): return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": iso(row["created_at"])} +def chat_view(row, members=None): + return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in (members or [])], "created_at": iso(row["created_at"])} +def message_view(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": iso(row["created_at"])} + +def error(request, status, code, message, headers=None): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + h = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID} + if headers: h.update(headers) + return JSONResponse({"error": {"code": code, "message": message, "request_id": rid}}, status_code=status, headers=h) + +@app.middleware("http") +async def request_context(request: Request, call_next): + request.state.request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex + try: + response = await call_next(request) + except Exception: + return error(request, 500, "internal_error", "Внутренняя ошибка") + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + +@app.on_event("startup") +def startup(): + pool.open(wait=True) + with pool.connection() as conn: + conn.execute("SELECT 1") + +@app.on_event("shutdown") +def shutdown(): pool.close() + +async def body(request): + try: value = await request.json() + except Exception: raise ValueError("malformed json") + if not isinstance(value, dict): raise ValueError("object expected") + return value +def require_fields(data, required, allowed): + if set(data) != set(required) if allowed is None else not set(data).issubset(set(allowed)) or not set(required).issubset(set(data)): raise ValueError("invalid fields") +def bearer(request): + value = request.headers.get("Authorization", "") + if not value.startswith("Bearer ") or len(value) < 40: return None + token = value[7:] + try: + raw = cache.get("session:" + token) + except Exception: raw = None + if not raw: return None + try: return json.loads(raw) + except Exception: return None +def auth(request): + session = bearer(request) + if not session: return None + with pool.connection() as conn: + return conn.execute("SELECT * FROM users WHERE id=%s", (session["user_id"],)).fetchone() +def auth_or_error(request): + user = auth(request) + if not user: return None, error(request, 401, "unauthorized", "Требуется действующая Bearer-сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'}) + return user, None +def member(conn, chat_id, user_id): + return conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone() is not None + +@app.get("/health/live") +def live(): return {"status": "ok"} +@app.get("/health/ready") +def ready(request: Request): + try: + with pool.connection() as conn: conn.execute("SELECT 1") + cache.ping() + return {"status": "ok"} + except Exception: return error(request, 503, "unavailable", "Зависимость недоступна") + +@app.get("/debug/cache-stats") +def cache_stats(request: Request): + if os.getenv("ENABLE_DEBUG_METRICS") != "1": return error(request, 404, "not_found", "Ресурс не найден") + return {"chat_list_sql_reads": cache_sql_reads} + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + try: + data = await body(request); require_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"]) + username, display, password = data["username"], data["display_name"], data["password"] + if not isinstance(username, str) or not __import__("re").fullmatch(r"[a-z0-9_]{3,32}", username) or not isinstance(display, str) or not display.strip() or not 1 <= len(display) <= 100 or not isinstance(password, str) or not 12 <= len(password) <= 128: raise ValueError("invalid fields") + user_id, created = uuid.uuid4(), now(); hashed = ph.hash(password) + with pool.connection() as conn: + try: conn.execute("INSERT INTO users VALUES (%s,%s,%s,%s,%s)", (user_id, username, display, hashed, created)); conn.commit() + except Exception as exc: + conn.rollback() + if "unique" in str(exc).lower(): return error(request, 409, "conflict", "Имя пользователя уже занято") + raise + return user_view({"id": user_id, "username": username, "display_name": display, "created_at": created}) + except ValueError: return error(request, 400, "invalid_request", "Неверные поля запроса") + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(request: Request): + header = request.headers.get("Authorization", "") + try: + if not header.startswith("Basic "): raise ValueError + decoded = base64.b64decode(header[6:], validate=True).decode("utf-8"); username, password = decoded.split(":", 1) + with pool.connection() as conn: row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() + if not row: raise ValueError + ph.verify(row["password_hash"], password) + except Exception: return error(request, 401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + token = secrets.token_urlsafe(32); expires = now() + timedelta(seconds=SESSION_TTL) + cache.setex("session:" + token, SESSION_TTL, json.dumps({"user_id": str(row["id"]), "expires_at": iso(expires)})) + return {"token": token, "token_type": "Bearer", "expires_at": iso(expires), "user": user_view(row)} + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + if not bearer(request): return error(request, 401, "unauthorized", "Сессия недействительна", {"WWW-Authenticate": 'Bearer realm="messenger"'}) + cache.delete("session:" + request.headers["Authorization"][7:]); return Response(status_code=204) + +@app.get("/api/v1/users/me") +def me(request: Request): + user, failure = auth_or_error(request) + if failure: return failure + return user_view(user) + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user, failure = auth_or_error(request) + if failure: return failure + try: + data = await body(request); require_fields(data, ["title", "member_ids"], ["title", "member_ids"]) + title, members = data["title"], data["member_ids"] + if not isinstance(title, str) or not title.strip() or not 1 <= len(title) <= 100 or not isinstance(members, list) or not 1 <= len(members) <= 100 or len(set(members)) != len(members): raise ValueError + ids = [uid(x) for x in members]; me_id = user["id"] + if me_id not in ids: raise ValueError + chat_id, created = uuid.uuid4(), now() + with pool.connection() as conn: + if conn.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s)", (ids,)).fetchone()["n"] != len(ids): raise ValueError + with conn.transaction(): + conn.execute("INSERT INTO chats VALUES (%s,%s,%s)", (chat_id,title,created)) + for member_id in ids: + conn.execute("INSERT INTO chat_members VALUES (%s,%s)", (chat_id, member_id)) + for x in ids: cache.delete("chats:" + str(x)) + return chat_view({"id": chat_id, "title": title, "created_at": created}, ids) + except ValueError: return error(request, 400, "invalid_request", "Неверные данные чата") + +@app.get("/api/v1/chats") +def list_chats(request: Request, limit: int = 50, cursor: str | None = None): + global cache_sql_reads + user, failure = auth_or_error(request) + if failure: return failure + if limit < 1 or limit > 100: return error(request, 400, "invalid_request", "limit должен быть от 1 до 100") + if cursor == "": return error(request, 400, "invalid_request", "Неверный cursor") + key = "chats:" + str(user["id"]) + after = None + if cursor: + try: + decoded = json.loads(base64.urlsafe_b64decode(cursor.encode() + b"=" * ((4 - len(cursor) % 4) % 4))) + if decoded[0] != "chats" or decoded[1] != str(user["id"]): raise ValueError + after = (datetime.fromisoformat(decoded[2].replace("Z", "+00:00")), uid(decoded[3])) + except Exception: return error(request, 400, "invalid_request", "Неверный cursor") + try: + cached = cache.get(key) + if cached: + result = json.loads(cached) + if after: result = [x for x in result if (datetime.fromisoformat(x["created_at"].replace("Z", "+00:00")), uuid.UUID(x["id"])) > after] + page = result[:limit] + next_cursor = None + if len(result) > limit and page: + tail = page[-1]; next_cursor = base64.urlsafe_b64encode(json.dumps(["chats", str(user["id"]), tail["created_at"], tail["id"]]).encode()).decode().rstrip("=") + return {"items": page, "next_cursor": next_cursor} + except Exception: pass + with chat_cache_lock: + try: cached = cache.get(key) + except Exception: cached = None + if cached: + result = json.loads(cached) + else: + with pool.connection() as conn: + rows = conn.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id", (user["id"],)).fetchall(); result=[] + for row in rows: + ids = [x["user_id"] for x in conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall()]; result.append(chat_view(row,ids)) + cache_sql_reads += 1 + try: cache.setex(key, 30, json.dumps(result)) + except Exception: pass + if after: result = [x for x in result if (datetime.fromisoformat(x["created_at"].replace("Z", "+00:00")), uuid.UUID(x["id"])) > after] + page = result[:limit] + next_cursor = None + if len(result) > limit and page: + tail = page[-1] + next_cursor = base64.urlsafe_b64encode(json.dumps(["chats", str(user["id"]), iso(datetime.fromisoformat(tail["created_at"].replace("Z", "+00:00"))), tail["id"]]).encode()).decode().rstrip("=") + return {"items": page, "next_cursor": next_cursor} + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + user, failure = auth_or_error(request) + if failure: return failure + try: cid=uid(chat_id) + except ValueError: return error(request,400,"invalid_request","Неверный UUID") + with pool.connection() as conn: + row=conn.execute("SELECT * FROM chats WHERE id=%s",(cid,)).fetchone() + if not row or not member(conn,cid,user["id"]): return error(request,404,"not_found","Чат не найден") + ids=[x["user_id"] for x in conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s",(cid,)).fetchall()] + return chat_view(row,ids) + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + user, failure = auth_or_error(request) + if failure: return failure + try: cid=uid(chat_id); data=await body(request); require_fields(data,["text"],["text"]); text=data["text"] + except ValueError: return error(request,400,"invalid_request","Неверные данные сообщения") + if not isinstance(text,str) or not text.strip() or not 1 <= len(text) <= 4000: return error(request,400,"invalid_request","Текст должен быть от 1 до 4000 символов") + with pool.connection() as conn: + if not member(conn,cid,user["id"]): return error(request,404,"not_found","Чат не найден") + mid, created=uuid.uuid4(),now(); conn.execute("INSERT INTO messages VALUES (%s,%s,%s,%s,%s)",(mid,cid,user["id"],text,created)); conn.commit() + return message_view({"id":mid,"chat_id":cid,"sender_id":user["id"],"text":text,"created_at":created}) + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request, limit: int = 50, cursor: str | None = None): + user, failure = auth_or_error(request) + if failure: return failure + if limit<1 or limit>100: return error(request,400,"invalid_request","limit должен быть от 1 до 100") + if cursor == "": return error(request,400,"invalid_request","Неверный cursor") + try: cid=uid(chat_id) + except ValueError: return error(request,400,"invalid_request","Неверный UUID") + last=None + if cursor: + try: + last=json.loads(base64.urlsafe_b64decode(cursor.encode()+b"="*((4-len(cursor)%4)%4))) + if last[0] != "messages" or last[1] != str(cid): raise ValueError + last = last[2:] + except Exception: return error(request,400,"invalid_request","Неверный cursor") + with pool.connection() as conn: + if not member(conn,cid,user["id"]): return error(request,404,"not_found","Чат не найден") + if last: + try: rows=conn.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s",(cid,datetime.fromisoformat(last[0].replace("Z","+00:00")),uid(last[1]),limit+1)).fetchall() + except Exception: return error(request,400,"invalid_request","Неверный cursor") + else: rows=conn.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s",(cid,limit+1)).fetchall() + more=len(rows)>limit; rows=rows[:limit]; next_cursor=None + if more: + tail=rows[-1]; next_cursor=base64.urlsafe_b64encode(json.dumps(["messages", str(cid), iso(tail["created_at"]),str(tail["id"])]).encode()).decode().rstrip("=") + return {"items":[message_view(x) for x in rows],"next_cursor":next_cursor} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..b3ed65e3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,31 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-local-only-password} + volumes: ["postgres_data:/var/lib/postgresql/data"] + healthcheck: {test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"], interval: 3s, timeout: 3s, retries: 20} + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: ["redis_data:/data"] + healthcheck: {test: ["CMD", "redis-cli", "ping"], interval: 3s, timeout: 3s, retries: 20} + api: + build: . + image: mtusi-messenger-lab2:2.0 + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:-local-only-password}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + INSTANCE_ID: api-1 + ENABLE_DEBUG_METRICS: ${ENABLE_DEBUG_METRICS:-1} + ports: ["8080:8080"] + depends_on: + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + healthcheck: {test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"], interval: 5s, timeout: 3s, retries: 20} +volumes: + postgres_data: + redis_data: diff --git a/evidence/explain_history.txt b/evidence/explain_history.txt new file mode 100644 index 00000000..037ceafb --- /dev/null +++ b/evidence/explain_history.txt @@ -0,0 +1,13 @@ + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.29..4.20 rows=50 width=78) (actual time=0.015..0.025 rows=50 loops=1) + Buffers: shared hit=3 + -> Index Scan using messages_history_idx on messages (cost=0.29..782.88 rows=10002 width=78) (actual time=0.014..0.019 rows=50 loops=1) + Index Cond: (chat_id = 'c66537d0-8434-4560-b135-3fee9c032f46'::uuid) + Buffers: shared hit=3 + Planning: + Buffers: shared hit=107 + Planning Time: 0.418 ms + Execution Time: 0.045 ms +(9 rows) + diff --git a/evidence/track_cache.json b/evidence/track_cache.json new file mode 100644 index 00000000..363b4a97 --- /dev/null +++ b/evidence/track_cache.json @@ -0,0 +1,9 @@ +{ + "hypothesis": "a lock prevents concurrent cache stampede and invalidation preserves read-after-write", + "cold_ms": 3.221, + "warm_p50_ms": 1.992, + "warm_p95_ms": 2.35, + "sql_reads_for_20_concurrent_cold_requests": 1, + "sql_reads_after_redis_restart": 1, + "read_after_write": "POST /chats deletes chats:" +} diff --git a/evidence/track_cache.py b/evidence/track_cache.py new file mode 100755 index 00000000..7e25f224 --- /dev/null +++ b/evidence/track_cache.py @@ -0,0 +1,30 @@ +"""Трек 1: конкурентный cache stampede и восстановление после Redis restart.""" +import argparse, base64, json, statistics, subprocess, time, urllib.request, uuid +from concurrent.futures import ThreadPoolExecutor + +def call(url, method="GET", body=None, headers=None): + data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None + req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + with urllib.request.urlopen(req) as r: return json.loads(r.read()) +def main(): + p=argparse.ArgumentParser(); p.add_argument("--base-url",default="http://localhost:8080"); a=p.parse_args(); b=a.base_url + name="research_"+uuid.uuid4().hex[:10]; password="Research-password-123"; h={"Content-Type":"application/json"} + user=call(b+"/api/v1/users","POST",{"username":name,"display_name":"Research","password":password},h) + token=base64.b64encode((name+":"+password).encode()).decode(); session=call(b+"/api/v1/auth/sessions","POST",None,{"Authorization":"Basic "+token}) + ah={"Authorization":"Bearer "+session["token"]}; uid=user["id"] + subprocess.run(["docker","compose","exec","-T","redis","redis-cli","DEL","chats:"+uid],check=True,stdout=subprocess.DEVNULL) + before=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"] + cold=time.perf_counter(); call(b+"/api/v1/chats",headers=ah); cold=(time.perf_counter()-cold)*1000 + warm=[] + for _ in range(30): + t=time.perf_counter(); call(b+"/api/v1/chats",headers=ah); warm.append((time.perf_counter()-t)*1000) + warmed=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"] + subprocess.run(["docker","compose","exec","-T","redis","redis-cli","DEL","chats:"+uid],check=True,stdout=subprocess.DEVNULL) + with ThreadPoolExecutor(max_workers=20) as pool: list(pool.map(lambda _: call(b+"/api/v1/chats",headers=ah), range(20))) + concurrent=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"]-warmed + subprocess.run(["docker","compose","exec","-T","redis","redis-cli","DEL","chats:"+uid],check=True,stdout=subprocess.DEVNULL) + subprocess.run(["docker","compose","restart","redis"],check=True,stdout=subprocess.DEVNULL) + time.sleep(2); call(b+"/api/v1/chats",headers=ah) + after_restart=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"]-warmed-concurrent + print(json.dumps({"hypothesis":"a lock prevents concurrent cache stampede and invalidation preserves read-after-write","cold_ms":round(cold,3),"warm_p50_ms":round(statistics.median(warm),3),"warm_p95_ms":round(sorted(warm)[28],3),"sql_reads_for_20_concurrent_cold_requests":concurrent,"sql_reads_after_redis_restart":after_restart,"read_after_write":"POST /chats deletes chats:"},ensure_ascii=False,indent=2)) +if __name__=="__main__": main() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..09678c0e --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, username VARCHAR(32) UNIQUE NOT NULL, + display_name VARCHAR(100) NOT NULL, password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, title VARCHAR(100) NOT NULL, created_at TIMESTAMPTZ NOT NULL +); +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX IF NOT EXISTS messages_history_idx ON messages(chat_id, created_at, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..265210d8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +psycopg[binary,pool]==3.2.3 +redis==5.2.1 +argon2-cffi==23.1.0 diff --git a/scripts/explain_history.sql b/scripts/explain_history.sql new file mode 100644 index 00000000..52e548ab --- /dev/null +++ b/scripts/explain_history.sql @@ -0,0 +1,7 @@ +-- run inside postgres after loading >=10000 synthetic messages +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = :'chat_id' +ORDER BY created_at, id +LIMIT 50; diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100755 index 00000000..07a01aa8 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,10 @@ +import os +import psycopg +from pathlib import Path + +url = os.getenv("DATABASE_URL", "postgresql://messenger:messenger@postgres:5432/messenger") +with psycopg.connect(url) as conn: + for path in sorted(Path("migrations").glob("*.sql")): + conn.execute(path.read_text()) + conn.commit() +print("migrations applied") diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..807fd622 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +# Перезапускаем API и оба stateful-сервиса, не удаляя volumes. +docker compose up -d --build --remove-orphans +docker compose restart postgres redis api +docker compose ps diff --git a/tests/level4.py b/tests/level4.py new file mode 100644 index 00000000..8d6947e7 --- /dev/null +++ b/tests/level4.py @@ -0,0 +1,29 @@ +"""Небольшая проверка требований уровня 4.""" +import base64, json, os, threading, time, urllib.error, urllib.request, uuid + +BASE = os.getenv("BASE_URL", "http://localhost:8080") +def call(method, path, body=None, headers=None): + data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None + h = {"Content-Type":"application/json"} if body is not None else {}; h.update(headers or {}) + req = urllib.request.Request(BASE+path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=5) as r: return r.status, json.loads(r.read() or b"{}") + except urllib.error.HTTPError as e: return e.code, json.loads(e.read() or b"{}") +def main(): + username = "race_"+uuid.uuid4().hex[:12]; password="Level4-password-123"; body={"username":username,"display_name":"Race","password":password} + results=[] + def register(): results.append(call("POST","/api/v1/users",body)[0]) + threads=[threading.Thread(target=register) for _ in range(8)] + for t in threads:t.start() + for t in threads:t.join() + assert results.count(201)==1 and results.count(409)==7, results + token=base64.b64encode((username+":"+password).encode()).decode(); status,session=call("POST","/api/v1/auth/sessions",headers={"Authorization":"Basic "+token}); assert status==201 + auth={"Authorization":"Bearer "+session["token"]}; assert call("GET","/api/v1/users/me",headers=auth)[0]==200 + rollback_title = "rollback_"+uuid.uuid4().hex[:8] + assert call("POST", "/api/v1/chats", {"title": rollback_title, "member_ids": ["00000000-0000-4000-8000-000000000000"]}, auth)[0] == 400 + listed = call("GET", "/api/v1/chats", headers=auth)[1] + assert all(item["title"] != rollback_title for item in listed["items"]) + if int(os.getenv("SESSION_TTL_SECONDS","3600")) <= 3: + time.sleep(3); assert call("GET","/api/v1/users/me",headers=auth)[0]==401 + print("PASS: registration race and optional short-TTL expiry") +if __name__ == "__main__": main() -- 2.54.0 From 1544277c0da436d54b8cea17758506159549d70f Mon Sep 17 00:00:00 2001 From: Nik <121254636+nikikiki19@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:51:56 +0300 Subject: [PATCH 2/5] =?UTF-8?q?=D0=A3=D1=82=D0=BE=D1=87=D0=BD=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D1=81=D1=81=D1=8B=D0=BB=D0=BA=D1=83=20=D0=BD=D0=B0=20?= =?UTF-8?q?MR=20=D0=B2=20=D0=BE=D1=82=D1=87=D1=91=D1=82=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 1d216570..d2659b34 100644 --- a/REPORT.md +++ b/REPORT.md @@ -4,7 +4,9 @@ - Группа: БВТ2304 - Целевая оценка: 5 - Выбранный трек 5: трек 1, согласованность кэша -- Ветка/Commit/PR: заполняется после публикации +- Ветка: `solution/bvt2304-borzov` +- Commit: `f87929c` +- MR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab2/pulls/28 В этой работе я продолжил REST-мессенджер из лабораторной 1. In-memory-хранилище заменено на PostgreSQL, добавлены пароль при регистрации, Basic-аутентификация только для входа и непрозрачные Bearer-сессии. Redis используется для TTL-сессий и кэша списка чатов. Сессии записываются в Redis с AOF и переживают перезапуск API и Redis-volume. -- 2.54.0 From 071d73614c34cd696642da2be1832fe8eb2496ea Mon Sep 17 00:00:00 2001 From: Nik <121254636+nikikiki19@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:23:31 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=D0=A3=D0=BF=D1=80=D0=BE=D1=81=D1=82=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=BA=D0=BE=D0=B4=20API=20=D0=B8=20=D0=BE=D0=B1?= =?UTF-8?q?=D0=BD=D0=BE=D0=B2=D0=B8=D1=82=D1=8C=20=D0=B4=D0=BE=D0=BA=D0=B0?= =?UTF-8?q?=D0=B7=D0=B0=D1=82=D0=B5=D0=BB=D1=8C=D1=81=D1=82=D0=B2=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 4 +- app/main.py | 647 +++++++++++++++++++++++++------------- evidence/track_cache.json | 6 +- 3 files changed, 438 insertions(+), 219 deletions(-) diff --git a/REPORT.md b/REPORT.md index d2659b34..55bb28e7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -6,7 +6,7 @@ - Выбранный трек 5: трек 1, согласованность кэша - Ветка: `solution/bvt2304-borzov` - Commit: `f87929c` -- MR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab2/pulls/28 +- MR: заполняется после публикации новой версии В этой работе я продолжил REST-мессенджер из лабораторной 1. In-memory-хранилище заменено на PostgreSQL, добавлены пароль при регистрации, Basic-аутентификация только для входа и непрозрачные Bearer-сессии. Redis используется для TTL-сессий и кэша списка чатов. Сессии записываются в Redis с AOF и переживают перезапуск API и Redis-volume. @@ -59,7 +59,7 @@ PostgreSQL является источником истины для польз Гипотеза: Redis-кэш уменьшит задержку повторного чтения списка чатов, а явная инвалидация после создания чата сохранит read-after-write. -Методика: один синтетический пользователь на локальном Docker Compose, холодный `GET /api/v1/chats`, 30 последовательных чтений и 20 одновременных чтений после удаления ключа. Счётчик SQL-запросов включён только в локальном debug-режиме. В последнем прогоне получены cold 3.221 ms, warm p50 1.992 ms, warm p95 2.350 ms; 20 конкурентных запросов выполнили 1 SQL-чтение благодаря блокировке, после удаления ключа и рестарта Redis восстановление выполнило 1 SQL-чтение. После `POST /api/v1/chats` ключ пользователя удаляется; следующий GET заново читает PostgreSQL и возвращает новый чат. Ограничение измерения — один ноутбук и небольшой синтетический набор данных, поэтому результат показывает поведение защиты от stampede и направление задержки, а не универсальный RPS. +Методика: один синтетический пользователь на локальном Docker Compose, холодный `GET /api/v1/chats`, 30 последовательных чтений и 20 одновременных чтений после удаления ключа. Счётчик SQL-запросов включён только в локальном debug-режиме. В последнем прогоне получены cold 5.189 ms, warm p50 2.104 ms, warm p95 2.530 ms; 20 конкурентных запросов выполнили 1 SQL-чтение благодаря блокировке, после удаления ключа и рестарта Redis восстановление выполнило 1 SQL-чтение. После `POST /api/v1/chats` ключ пользователя удаляется; следующий GET заново читает PostgreSQL и возвращает новый чат. Ограничение измерения — один ноутбук и небольшой синтетический набор данных, поэтому результат показывает поведение защиты от stampede и направление задержки, а не универсальный RPS. ## Эксперименты и отказы diff --git a/app/main.py b/app/main.py index 04bc83f1..69886640 100644 --- a/app/main.py +++ b/app/main.py @@ -1,7 +1,11 @@ -import base64, hashlib, json, os, secrets, threading, uuid -from datetime import datetime, timezone, timedelta -from pathlib import Path -from typing import Any +import base64 +import json +import os +import re +import secrets +import threading +import uuid +from datetime import datetime, timedelta, timezone import redis from argon2 import PasswordHasher @@ -10,253 +14,468 @@ from fastapi.responses import JSONResponse, Response from psycopg.rows import dict_row from psycopg_pool import ConnectionPool -DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://messenger:messenger@postgres:5432/messenger") -REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") -SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600")) -INSTANCE_ID = os.getenv("INSTANCE_ID", uuid.uuid4().hex[:12]) -ph = PasswordHasher() -pool = ConnectionPool(DATABASE_URL, min_size=1, max_size=int(os.getenv("DB_POOL_SIZE", "10")), timeout=5, kwargs={"row_factory": dict_row}, open=False) -cache = redis.Redis.from_url(REDIS_URL, decode_responses=True, socket_connect_timeout=1) +DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger') +REDIS_URL = os.getenv('REDIS_URL', 'redis://redis:6379/0') +SESSION_TTL_SECONDS = int(os.getenv('SESSION_TTL_SECONDS', '3600')) +INSTANCE_ID = os.getenv('INSTANCE_ID', uuid.uuid4().hex[:12]) +DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', '10')) + +password_hasher = PasswordHasher() +db_pool = ConnectionPool( + DATABASE_URL, min_size=1, max_size=DB_POOL_SIZE, timeout=5, + kwargs={'row_factory': dict_row}, open=False, +) +redis_client = redis.Redis.from_url( + REDIS_URL, decode_responses=True, socket_connect_timeout=1, +) chat_cache_lock = threading.Lock() -cache_sql_reads = 0 -app = FastAPI(title="MTUSI Messenger Lab 2", version="1.2.0") +chat_cache_sql_reads = 0 +app = FastAPI(title='MTUSI Messenger Lab 2', version='1.2.0') -def now(): return datetime.now(timezone.utc) -def iso(value): return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") -def uid(value): - try: return uuid.UUID(str(value)) - except (ValueError, TypeError): raise ValueError("invalid uuid") -def user_view(row): return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": iso(row["created_at"])} -def chat_view(row, members=None): - return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in (members or [])], "created_at": iso(row["created_at"])} -def message_view(row): - return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": iso(row["created_at"])} -def error(request, status, code, message, headers=None): - rid = getattr(request.state, "request_id", uuid.uuid4().hex) - h = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID} - if headers: h.update(headers) - return JSONResponse({"error": {"code": code, "message": message, "request_id": rid}}, status_code=status, headers=h) +def utc_now(): + return datetime.now(timezone.utc) -@app.middleware("http") + +def format_time(value): + return value.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z') + + +def parse_uuid(value): + try: + return uuid.UUID(str(value)) + except (TypeError, ValueError) as exc: + raise ValueError('invalid uuid') from exc + + +def user_response(row): + return { + 'id': str(row['id']), 'username': row['username'], + 'display_name': row['display_name'], 'created_at': format_time(row['created_at']), + } + + +def chat_response(row, member_ids): + return { + 'id': str(row['id']), 'title': row['title'], + 'member_ids': [str(item) for item in member_ids], + 'created_at': format_time(row['created_at']), + } + + +def message_response(row): + return { + 'id': str(row['id']), 'chat_id': str(row['chat_id']), + 'sender_id': str(row['sender_id']), 'text': row['text'], + 'created_at': format_time(row['created_at']), + } + + +def error_response(request, status_code, code, message, extra_headers=None): + request_id = getattr(request.state, 'request_id', uuid.uuid4().hex) + headers = {'X-Request-Id': request_id, 'X-Instance-Id': INSTANCE_ID} + if extra_headers: + headers.update(extra_headers) + return JSONResponse( + status_code=status_code, + content={'error': {'code': code, 'message': message, 'request_id': request_id}}, + headers=headers, + ) + + +@app.middleware('http') async def request_context(request: Request, call_next): - request.state.request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = request.headers.get('X-Request-Id') or uuid.uuid4().hex try: response = await call_next(request) except Exception: - return error(request, 500, "internal_error", "Внутренняя ошибка") - response.headers["X-Request-Id"] = request.state.request_id - response.headers["X-Instance-Id"] = INSTANCE_ID + return error_response(request, 500, 'internal_error', 'Внутренняя ошибка') + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = INSTANCE_ID return response -@app.on_event("startup") -def startup(): - pool.open(wait=True) - with pool.connection() as conn: - conn.execute("SELECT 1") -@app.on_event("shutdown") -def shutdown(): pool.close() +@app.on_event('startup') +def start_application(): + db_pool.open(wait=True) + with db_pool.connection() as connection: + connection.execute('SELECT 1') -async def body(request): - try: value = await request.json() - except Exception: raise ValueError("malformed json") - if not isinstance(value, dict): raise ValueError("object expected") + +@app.on_event('shutdown') +def stop_application(): + db_pool.close() + + +async def read_json_body(request): + try: + value = await request.json() + except Exception as exc: + raise ValueError('malformed json') from exc + if not isinstance(value, dict): + raise ValueError('request body must be an object') return value -def require_fields(data, required, allowed): - if set(data) != set(required) if allowed is None else not set(data).issubset(set(allowed)) or not set(required).issubset(set(data)): raise ValueError("invalid fields") -def bearer(request): - value = request.headers.get("Authorization", "") - if not value.startswith("Bearer ") or len(value) < 40: return None - token = value[7:] + + +def require_fields(data, required): + if set(data) != required: + raise ValueError('unexpected or missing fields') + + +def get_bearer_session(request): + authorization = request.headers.get('Authorization', '') + if not authorization.startswith('Bearer '): + return None + token = authorization.removeprefix('Bearer ') + if len(token) < 40: + return None try: - raw = cache.get("session:" + token) - except Exception: raw = None - if not raw: return None - try: return json.loads(raw) - except Exception: return None -def auth(request): - session = bearer(request) - if not session: return None - with pool.connection() as conn: - return conn.execute("SELECT * FROM users WHERE id=%s", (session["user_id"],)).fetchone() -def auth_or_error(request): - user = auth(request) - if not user: return None, error(request, 401, "unauthorized", "Требуется действующая Bearer-сессия", {"WWW-Authenticate": 'Bearer realm="messenger"'}) + value = redis_client.get('session:' + token) + return json.loads(value) if value else None + except Exception: + return None + + +def get_current_user(request): + session = get_bearer_session(request) + if session is None: + return None + with db_pool.connection() as connection: + return connection.execute( + 'SELECT * FROM users WHERE id=%s', (session['user_id'],) + ).fetchone() + + +def require_current_user(request): + user = get_current_user(request) + if user is None: + return None, error_response( + request, 401, 'unauthorized', 'Требуется действующая Bearer-сессия', + {'WWW-Authenticate': 'Bearer realm="messenger"'}, + ) return user, None -def member(conn, chat_id, user_id): - return conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone() is not None -@app.get("/health/live") -def live(): return {"status": "ok"} -@app.get("/health/ready") -def ready(request: Request): + +def is_chat_member(connection, chat_id, user_id): + row = connection.execute( + 'SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', + (chat_id, user_id), + ).fetchone() + return row is not None + + +def make_cursor(scope, owner_id, row): + created_at = row['created_at'] + if isinstance(created_at, datetime): + created_at = format_time(created_at) + data = [scope, owner_id, created_at, str(row['id'])] + return base64.urlsafe_b64encode(json.dumps(data).encode()).decode().rstrip('=') + + +def read_cursor(cursor, expected_scope, expected_owner): try: - with pool.connection() as conn: conn.execute("SELECT 1") - cache.ping() - return {"status": "ok"} - except Exception: return error(request, 503, "unavailable", "Зависимость недоступна") + padding = '=' * ((4 - len(cursor) % 4) % 4) + value = json.loads(base64.urlsafe_b64decode((cursor + padding).encode())) + if value[0] != expected_scope or value[1] != expected_owner: + raise ValueError + timestamp = datetime.fromisoformat(value[2].replace('Z', '+00:00')) + return timestamp, parse_uuid(value[3]) + except (IndexError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise ValueError('invalid cursor') from exc -@app.get("/debug/cache-stats") + +def page_chats(items, limit, cursor_data, owner_id): + if cursor_data: + items = [ + item for item in items + if ( + datetime.fromisoformat(item['created_at'].replace('Z', '+00:00')), + uuid.UUID(item['id']), + ) > cursor_data + ] + page = items[:limit] + next_cursor = make_cursor('chats', owner_id, page[-1]) if len(items) > limit and page else None + return {'items': page, 'next_cursor': next_cursor} + + +@app.get('/health/live') +def health_live(): + return {'status': 'ok'} + + +@app.get('/health/ready') +def health_ready(request: Request): + try: + with db_pool.connection() as connection: + connection.execute('SELECT 1') + redis_client.ping() + except Exception: + return error_response(request, 503, 'unavailable', 'Зависимость недоступна') + return {'status': 'ok'} + + +@app.get('/debug/cache-stats') def cache_stats(request: Request): - if os.getenv("ENABLE_DEBUG_METRICS") != "1": return error(request, 404, "not_found", "Ресурс не найден") - return {"chat_list_sql_reads": cache_sql_reads} + if os.getenv('ENABLE_DEBUG_METRICS') != '1': + return error_response(request, 404, 'not_found', 'Ресурс не найден') + return {'chat_list_sql_reads': chat_cache_sql_reads} -@app.post("/api/v1/users", status_code=201) + +@app.post('/api/v1/users', status_code=201) async def create_user(request: Request): try: - data = await body(request); require_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"]) - username, display, password = data["username"], data["display_name"], data["password"] - if not isinstance(username, str) or not __import__("re").fullmatch(r"[a-z0-9_]{3,32}", username) or not isinstance(display, str) or not display.strip() or not 1 <= len(display) <= 100 or not isinstance(password, str) or not 12 <= len(password) <= 128: raise ValueError("invalid fields") - user_id, created = uuid.uuid4(), now(); hashed = ph.hash(password) - with pool.connection() as conn: - try: conn.execute("INSERT INTO users VALUES (%s,%s,%s,%s,%s)", (user_id, username, display, hashed, created)); conn.commit() + data = await read_json_body(request) + require_fields(data, {'username', 'display_name', 'password'}) + username, display_name, password = data['username'], data['display_name'], data['password'] + if not isinstance(username, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', username): + raise ValueError('invalid username') + if not isinstance(display_name, str) or not display_name.strip() or not 1 <= len(display_name) <= 100: + raise ValueError('invalid display name') + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise ValueError('invalid password') + + user_id, created_at = uuid.uuid4(), utc_now() + password_hash = password_hasher.hash(password) + with db_pool.connection() as connection: + try: + connection.execute( + 'INSERT INTO users VALUES (%s,%s,%s,%s,%s)', + (user_id, username, display_name, password_hash, created_at), + ) + connection.commit() except Exception as exc: - conn.rollback() - if "unique" in str(exc).lower(): return error(request, 409, "conflict", "Имя пользователя уже занято") + connection.rollback() + if 'unique' in str(exc).lower(): + return error_response(request, 409, 'conflict', 'Имя пользователя уже занято') raise - return user_view({"id": user_id, "username": username, "display_name": display, "created_at": created}) - except ValueError: return error(request, 400, "invalid_request", "Неверные поля запроса") + return user_response({'id': user_id, 'username': username, 'display_name': display_name, 'created_at': created_at}) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверные поля запроса') -@app.post("/api/v1/auth/sessions", status_code=201) + +@app.post('/api/v1/auth/sessions', status_code=201) def create_session(request: Request): - header = request.headers.get("Authorization", "") + authorization = request.headers.get('Authorization', '') try: - if not header.startswith("Basic "): raise ValueError - decoded = base64.b64decode(header[6:], validate=True).decode("utf-8"); username, password = decoded.split(":", 1) - with pool.connection() as conn: row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() - if not row: raise ValueError - ph.verify(row["password_hash"], password) - except Exception: return error(request, 401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) - token = secrets.token_urlsafe(32); expires = now() + timedelta(seconds=SESSION_TTL) - cache.setex("session:" + token, SESSION_TTL, json.dumps({"user_id": str(row["id"]), "expires_at": iso(expires)})) - return {"token": token, "token_type": "Bearer", "expires_at": iso(expires), "user": user_view(row)} + if not authorization.startswith('Basic '): + raise ValueError + credentials = base64.b64decode(authorization.removeprefix('Basic '), validate=True).decode() + username, password = credentials.split(':', 1) + with db_pool.connection() as connection: + user = connection.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone() + if user is None: + raise ValueError + password_hasher.verify(user['password_hash'], password) + except Exception: + return error_response( + request, 401, 'unauthorized', 'Неверные учётные данные', + {'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'}, + ) -@app.delete("/api/v1/auth/sessions/current", status_code=204) + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + redis_client.setex( + 'session:' + token, + SESSION_TTL_SECONDS, + json.dumps({'user_id': str(user['id']), 'expires_at': format_time(expires_at)}), + ) + return { + 'token': token, 'token_type': 'Bearer', + 'expires_at': format_time(expires_at), 'user': user_response(user), + } + + +@app.delete('/api/v1/auth/sessions/current', status_code=204) def logout(request: Request): - if not bearer(request): return error(request, 401, "unauthorized", "Сессия недействительна", {"WWW-Authenticate": 'Bearer realm="messenger"'}) - cache.delete("session:" + request.headers["Authorization"][7:]); return Response(status_code=204) + if get_bearer_session(request) is None: + return error_response( + request, 401, 'unauthorized', 'Сессия недействительна', + {'WWW-Authenticate': 'Bearer realm="messenger"'}, + ) + token = request.headers['Authorization'].removeprefix('Bearer ') + redis_client.delete('session:' + token) + return Response(status_code=204) -@app.get("/api/v1/users/me") -def me(request: Request): - user, failure = auth_or_error(request) - if failure: return failure - return user_view(user) -@app.post("/api/v1/chats", status_code=201) +@app.get('/api/v1/users/me') +def get_me(request: Request): + user, failure = require_current_user(request) + return failure if failure else user_response(user) + + +@app.post('/api/v1/chats', status_code=201) async def create_chat(request: Request): - user, failure = auth_or_error(request) - if failure: return failure + user, failure = require_current_user(request) + if failure: + return failure try: - data = await body(request); require_fields(data, ["title", "member_ids"], ["title", "member_ids"]) - title, members = data["title"], data["member_ids"] - if not isinstance(title, str) or not title.strip() or not 1 <= len(title) <= 100 or not isinstance(members, list) or not 1 <= len(members) <= 100 or len(set(members)) != len(members): raise ValueError - ids = [uid(x) for x in members]; me_id = user["id"] - if me_id not in ids: raise ValueError - chat_id, created = uuid.uuid4(), now() - with pool.connection() as conn: - if conn.execute("SELECT count(*) AS n FROM users WHERE id = ANY(%s)", (ids,)).fetchone()["n"] != len(ids): raise ValueError - with conn.transaction(): - conn.execute("INSERT INTO chats VALUES (%s,%s,%s)", (chat_id,title,created)) - for member_id in ids: - conn.execute("INSERT INTO chat_members VALUES (%s,%s)", (chat_id, member_id)) - for x in ids: cache.delete("chats:" + str(x)) - return chat_view({"id": chat_id, "title": title, "created_at": created}, ids) - except ValueError: return error(request, 400, "invalid_request", "Неверные данные чата") + data = await read_json_body(request) + require_fields(data, {'title', 'member_ids'}) + title, member_values = data['title'], data['member_ids'] + if not isinstance(title, str) or not title.strip() or not 1 <= len(title) <= 100: + raise ValueError('invalid title') + if not isinstance(member_values, list) or not 1 <= len(member_values) <= 100: + raise ValueError('invalid members') + if len(set(member_values)) != len(member_values): + raise ValueError('duplicate members') + member_ids = [parse_uuid(value) for value in member_values] + if user['id'] not in member_ids: + raise ValueError('current user must be a member') -@app.get("/api/v1/chats") + chat_id, created_at = uuid.uuid4(), utc_now() + with db_pool.connection() as connection: + count = connection.execute( + 'SELECT count(*) AS n FROM users WHERE id = ANY(%s)', (member_ids,) + ).fetchone()['n'] + if count != len(member_ids): + raise ValueError('unknown member') + with connection.transaction(): + connection.execute('INSERT INTO chats VALUES (%s,%s,%s)', (chat_id, title, created_at)) + for member_id in member_ids: + connection.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat_id, member_id)) + for member_id in member_ids: + try: + redis_client.delete('chats:' + str(member_id)) + except Exception: + pass + return chat_response({'id': chat_id, 'title': title, 'created_at': created_at}, member_ids) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверные данные чата') + + +@app.get('/api/v1/chats') def list_chats(request: Request, limit: int = 50, cursor: str | None = None): - global cache_sql_reads - user, failure = auth_or_error(request) - if failure: return failure - if limit < 1 or limit > 100: return error(request, 400, "invalid_request", "limit должен быть от 1 до 100") - if cursor == "": return error(request, 400, "invalid_request", "Неверный cursor") - key = "chats:" + str(user["id"]) - after = None - if cursor: + global chat_cache_sql_reads + user, failure = require_current_user(request) + if failure: + return failure + if not 1 <= limit <= 100: + return error_response(request, 400, 'invalid_request', 'limit должен быть от 1 до 100') + + cursor_data = None + if cursor is not None: + if not cursor: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') try: - decoded = json.loads(base64.urlsafe_b64decode(cursor.encode() + b"=" * ((4 - len(cursor) % 4) % 4))) - if decoded[0] != "chats" or decoded[1] != str(user["id"]): raise ValueError - after = (datetime.fromisoformat(decoded[2].replace("Z", "+00:00")), uid(decoded[3])) - except Exception: return error(request, 400, "invalid_request", "Неверный cursor") + cursor_data = read_cursor(cursor, 'chats', str(user['id'])) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') + + cache_key = 'chats:' + str(user['id']) try: - cached = cache.get(key) + cached = redis_client.get(cache_key) if cached: - result = json.loads(cached) - if after: result = [x for x in result if (datetime.fromisoformat(x["created_at"].replace("Z", "+00:00")), uuid.UUID(x["id"])) > after] - page = result[:limit] - next_cursor = None - if len(result) > limit and page: - tail = page[-1]; next_cursor = base64.urlsafe_b64encode(json.dumps(["chats", str(user["id"]), tail["created_at"], tail["id"]]).encode()).decode().rstrip("=") - return {"items": page, "next_cursor": next_cursor} - except Exception: pass + return page_chats(json.loads(cached), limit, cursor_data, str(user['id'])) + except Exception: + pass + with chat_cache_lock: - try: cached = cache.get(key) - except Exception: cached = None - if cached: - result = json.loads(cached) - else: - with pool.connection() as conn: - rows = conn.execute("SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s ORDER BY c.created_at,c.id", (user["id"],)).fetchall(); result=[] - for row in rows: - ids = [x["user_id"] for x in conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (row["id"],)).fetchall()]; result.append(chat_view(row,ids)) - cache_sql_reads += 1 - try: cache.setex(key, 30, json.dumps(result)) - except Exception: pass - if after: result = [x for x in result if (datetime.fromisoformat(x["created_at"].replace("Z", "+00:00")), uuid.UUID(x["id"])) > after] - page = result[:limit] - next_cursor = None - if len(result) > limit and page: - tail = page[-1] - next_cursor = base64.urlsafe_b64encode(json.dumps(["chats", str(user["id"]), iso(datetime.fromisoformat(tail["created_at"].replace("Z", "+00:00"))), tail["id"]]).encode()).decode().rstrip("=") - return {"items": page, "next_cursor": next_cursor} - -@app.get("/api/v1/chats/{chat_id}") -def get_chat(chat_id: str, request: Request): - user, failure = auth_or_error(request) - if failure: return failure - try: cid=uid(chat_id) - except ValueError: return error(request,400,"invalid_request","Неверный UUID") - with pool.connection() as conn: - row=conn.execute("SELECT * FROM chats WHERE id=%s",(cid,)).fetchone() - if not row or not member(conn,cid,user["id"]): return error(request,404,"not_found","Чат не найден") - ids=[x["user_id"] for x in conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s",(cid,)).fetchall()] - return chat_view(row,ids) - -@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) -async def create_message(chat_id: str, request: Request): - user, failure = auth_or_error(request) - if failure: return failure - try: cid=uid(chat_id); data=await body(request); require_fields(data,["text"],["text"]); text=data["text"] - except ValueError: return error(request,400,"invalid_request","Неверные данные сообщения") - if not isinstance(text,str) or not text.strip() or not 1 <= len(text) <= 4000: return error(request,400,"invalid_request","Текст должен быть от 1 до 4000 символов") - with pool.connection() as conn: - if not member(conn,cid,user["id"]): return error(request,404,"not_found","Чат не найден") - mid, created=uuid.uuid4(),now(); conn.execute("INSERT INTO messages VALUES (%s,%s,%s,%s,%s)",(mid,cid,user["id"],text,created)); conn.commit() - return message_view({"id":mid,"chat_id":cid,"sender_id":user["id"],"text":text,"created_at":created}) - -@app.get("/api/v1/chats/{chat_id}/messages") -def list_messages(chat_id: str, request: Request, limit: int = 50, cursor: str | None = None): - user, failure = auth_or_error(request) - if failure: return failure - if limit<1 or limit>100: return error(request,400,"invalid_request","limit должен быть от 1 до 100") - if cursor == "": return error(request,400,"invalid_request","Неверный cursor") - try: cid=uid(chat_id) - except ValueError: return error(request,400,"invalid_request","Неверный UUID") - last=None - if cursor: try: - last=json.loads(base64.urlsafe_b64decode(cursor.encode()+b"="*((4-len(cursor)%4)%4))) - if last[0] != "messages" or last[1] != str(cid): raise ValueError - last = last[2:] - except Exception: return error(request,400,"invalid_request","Неверный cursor") - with pool.connection() as conn: - if not member(conn,cid,user["id"]): return error(request,404,"not_found","Чат не найден") - if last: - try: rows=conn.execute("SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT %s",(cid,datetime.fromisoformat(last[0].replace("Z","+00:00")),uid(last[1]),limit+1)).fetchall() - except Exception: return error(request,400,"invalid_request","Неверный cursor") - else: rows=conn.execute("SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s",(cid,limit+1)).fetchall() - more=len(rows)>limit; rows=rows[:limit]; next_cursor=None - if more: - tail=rows[-1]; next_cursor=base64.urlsafe_b64encode(json.dumps(["messages", str(cid), iso(tail["created_at"]),str(tail["id"])]).encode()).decode().rstrip("=") - return {"items":[message_view(x) for x in rows],"next_cursor":next_cursor} + cached = redis_client.get(cache_key) + except Exception: + cached = None + if cached: + chats = json.loads(cached) + else: + with db_pool.connection() as connection: + rows = connection.execute( + 'SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s ORDER BY c.created_at,c.id', (user['id'],) + ).fetchall() + chats = [] + for row in rows: + members = connection.execute( + 'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],) + ).fetchall() + chats.append(chat_response(row, [item['user_id'] for item in members])) + chat_cache_sql_reads += 1 + try: + redis_client.setex(cache_key, 30, json.dumps(chats)) + except Exception: + pass + return page_chats(chats, limit, cursor_data, str(user['id'])) + + +@app.get('/api/v1/chats/{chat_id}') +def get_chat(chat_id: str, request: Request): + user, failure = require_current_user(request) + if failure: + return failure + try: + parsed_chat_id = parse_uuid(chat_id) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный UUID') + with db_pool.connection() as connection: + chat = connection.execute('SELECT * FROM chats WHERE id=%s', (parsed_chat_id,)).fetchone() + if chat is None or not is_chat_member(connection, parsed_chat_id, user['id']): + return error_response(request, 404, 'not_found', 'Чат не найден') + members = connection.execute('SELECT user_id FROM chat_members WHERE chat_id=%s', (parsed_chat_id,)).fetchall() + return chat_response(chat, [item['user_id'] for item in members]) + + +@app.post('/api/v1/chats/{chat_id}/messages', status_code=201) +async def create_message(chat_id: str, request: Request): + user, failure = require_current_user(request) + if failure: + return failure + try: + parsed_chat_id = parse_uuid(chat_id) + data = await read_json_body(request) + require_fields(data, {'text'}) + text = data['text'] + if not isinstance(text, str) or not text.strip() or not 1 <= len(text) <= 4000: + raise ValueError('invalid text') + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверные данные сообщения') + + with db_pool.connection() as connection: + if not is_chat_member(connection, parsed_chat_id, user['id']): + return error_response(request, 404, 'not_found', 'Чат не найден') + message_id, created_at = uuid.uuid4(), utc_now() + connection.execute( + 'INSERT INTO messages VALUES (%s,%s,%s,%s,%s)', + (message_id, parsed_chat_id, user['id'], text, created_at), + ) + connection.commit() + return message_response({'id': message_id, 'chat_id': parsed_chat_id, 'sender_id': user['id'], 'text': text, 'created_at': created_at}) + + +@app.get('/api/v1/chats/{chat_id}/messages') +def list_messages(chat_id: str, request: Request, limit: int = 50, cursor: str | None = None): + user, failure = require_current_user(request) + if failure: + return failure + if not 1 <= limit <= 100: + return error_response(request, 400, 'invalid_request', 'limit должен быть от 1 до 100') + try: + parsed_chat_id = parse_uuid(chat_id) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный UUID') + + cursor_data = None + if cursor is not None: + if not cursor: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') + try: + cursor_data = read_cursor(cursor, 'messages', str(parsed_chat_id)) + except ValueError: + return error_response(request, 400, 'invalid_request', 'Неверный cursor') + + with db_pool.connection() as connection: + if not is_chat_member(connection, parsed_chat_id, user['id']): + return error_response(request, 404, 'not_found', 'Чат не найден') + if cursor_data: + rows = connection.execute( + 'SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ' + 'ORDER BY created_at,id LIMIT %s', + (parsed_chat_id, cursor_data[0], cursor_data[1], limit + 1), + ).fetchall() + else: + rows = connection.execute( + 'SELECT * FROM messages WHERE chat_id=%s ORDER BY created_at,id LIMIT %s', + (parsed_chat_id, limit + 1), + ).fetchall() + + rows_to_return = rows[:limit] + next_cursor = make_cursor('messages', str(parsed_chat_id), rows_to_return[-1]) if len(rows) > limit else None + return {'items': [message_response(row) for row in rows_to_return], 'next_cursor': next_cursor} diff --git a/evidence/track_cache.json b/evidence/track_cache.json index 363b4a97..095dc2ca 100644 --- a/evidence/track_cache.json +++ b/evidence/track_cache.json @@ -1,8 +1,8 @@ { "hypothesis": "a lock prevents concurrent cache stampede and invalidation preserves read-after-write", - "cold_ms": 3.221, - "warm_p50_ms": 1.992, - "warm_p95_ms": 2.35, + "cold_ms": 5.189, + "warm_p50_ms": 2.104, + "warm_p95_ms": 2.53, "sql_reads_for_20_concurrent_cold_requests": 1, "sql_reads_after_redis_restart": 1, "read_after_write": "POST /chats deletes chats:" -- 2.54.0 From 7dbccb22062d87ef0dc267293ad145c21ace205a Mon Sep 17 00:00:00 2001 From: Nik <121254636+nikikiki19@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:24:30 +0300 Subject: [PATCH 4/5] =?UTF-8?q?=D0=A3=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D1=8C?= =?UTF-8?q?=20=D1=81=D1=81=D1=8B=D0=BB=D0=BA=D1=83=20=D0=BD=D0=B0=20=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D1=8B=D0=B9=20MR=20=D0=B2=20=D0=BE=D1=82=D1=87?= =?UTF-8?q?=D1=91=D1=82=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 55bb28e7..d382dde8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -6,7 +6,7 @@ - Выбранный трек 5: трек 1, согласованность кэша - Ветка: `solution/bvt2304-borzov` - Commit: `f87929c` -- MR: заполняется после публикации новой версии +- MR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab2/pulls/30 В этой работе я продолжил REST-мессенджер из лабораторной 1. In-memory-хранилище заменено на PostgreSQL, добавлены пароль при регистрации, Basic-аутентификация только для входа и непрозрачные Bearer-сессии. Redis используется для TTL-сессий и кэша списка чатов. Сессии записываются в Redis с AOF и переживают перезапуск API и Redis-volume. -- 2.54.0 From 132f1fb3e37d99616cd03e5626765ee6ab9acb21 Mon Sep 17 00:00:00 2001 From: Nik <121254636+nikikiki19@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:50:46 +0300 Subject: [PATCH 5/5] =?UTF-8?q?=D0=A3=D1=87=D0=B5=D1=81=D1=82=D1=8C=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=BC=D0=B5=D1=87=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BF?= =?UTF-8?q?=D0=BE=20=D1=82=D0=B0=D0=B9=D0=BC=D0=B0=D1=83=D1=82=D0=B0=D0=BC?= =?UTF-8?q?=20=D0=B8=20=D0=BA=D1=8D=D1=88=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 8 +-- app/main.py | 59 ++++++++++++------ evidence/track_cache.json | 13 ++-- evidence/track_cache.py | 124 ++++++++++++++++++++++++++++++-------- tests/level4.py | 5 +- 5 files changed, 156 insertions(+), 53 deletions(-) diff --git a/REPORT.md b/REPORT.md index d382dde8..49311366 100644 --- a/REPORT.md +++ b/REPORT.md @@ -50,16 +50,16 @@ PostgreSQL является источником истины для польз | Публичный smoke | `make test` | `PASS: lab2 public API scenario` | вывод команды | | PostgreSQL и миграции | `docker compose up -d --build` | API применяет `001_init.sql`, данные находятся в PostgreSQL | `migrations/001_init.sql` | | Сохранность API/БД/сессии | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | подтверждённое сообщение и прежний Bearer-токен сохранены | `scripts/restart.sh` | -| Redis с TTL и volume | `docker compose ps`, `redis-cli TTL session:*` | Redis healthy, TTL уменьшается, AOF включён | `compose.yaml` | +| Redis с TTL и volume | `docker compose ps`, `redis-cli TTL session:*` | Redis healthy, TTL уменьшается, AOF включён; операции ограничены socket timeout 1 s | `compose.yaml`, `app/main.py` | | Индекс истории | `EXPLAIN (ANALYZE, BUFFERS)` на 10 000 синтетических сообщениях | `Index Scan using messages_history_idx`, 50 строк за 0.045 ms | `evidence/explain_history.txt` | -| Уровень 4: race/rollback/expiry | `python tests/level4.py` при TTL=2 | 1 регистрация получила 201, 7 гонок — 409; невалидный чат не появился; просроченная сессия — 401 | `tests/level4.py` | -| Трек 5: кэш | `python evidence/track_cache.py` | 20 конкурентных холодных чтений дали 1 SQL-запрос; после очистки и рестарта Redis — 1 запрос на восстановление; запись инвалидирует ключ | `evidence/track_cache.json` | +| Уровень 4: race/rollback/expiry/limit | `python tests/level4.py`, запросы `limit=0/101/abc` | 1 регистрация получила 201, 7 гонок — 409; rollback выполняется внутри transaction; просроченная сессия и неверные limit возвращают 400/401 | `tests/level4.py` | +| Трек 5: кэш | `python evidence/track_cache.py` | baseline p50 4.469 ms, cached p50 1.862 ms, read-after-write=true, 20 конкурентных чтений — 1 SQL-запрос | `evidence/track_cache.json` | ## Исследовательский трек: согласованность кэша Гипотеза: Redis-кэш уменьшит задержку повторного чтения списка чатов, а явная инвалидация после создания чата сохранит read-after-write. -Методика: один синтетический пользователь на локальном Docker Compose, холодный `GET /api/v1/chats`, 30 последовательных чтений и 20 одновременных чтений после удаления ключа. Счётчик SQL-запросов включён только в локальном debug-режиме. В последнем прогоне получены cold 5.189 ms, warm p50 2.104 ms, warm p95 2.530 ms; 20 конкурентных запросов выполнили 1 SQL-чтение благодаря блокировке, после удаления ключа и рестарта Redis восстановление выполнило 1 SQL-чтение. После `POST /api/v1/chats` ключ пользователя удаляется; следующий GET заново читает PostgreSQL и возвращает новый чат. Ограничение измерения — один ноутбук и небольшой синтетический набор данных, поэтому результат показывает поведение защиты от stampede и направление задержки, а не универсальный RPS. +Методика: один синтетический пользователь на локальном Docker Compose. Сначала выполняются 10 чтений с удалением ключа перед каждым запросом — это baseline без кэша. Затем выполняются 30 чтений после прогрева Redis. В последнем прогоне baseline p50 составил 4.469 ms, cached p50 — 1.862 ms. После `POST /api/v1/chats` следующий GET сразу нашёл новый чат (`read_after_write=true`). Дополнительно 20 одновременных холодных чтений выполнили 1 SQL-запрос благодаря блокировке; после очистки и рестарта Redis кэш снова заполняется из PostgreSQL. Ограничение измерения — один ноутбук и небольшой синтетический набор данных, поэтому результат показывает поведение защиты и направление задержки, а не универсальный RPS. ## Эксперименты и отказы diff --git a/app/main.py b/app/main.py index 69886640..f054f4c0 100644 --- a/app/main.py +++ b/app/main.py @@ -26,10 +26,15 @@ db_pool = ConnectionPool( kwargs={'row_factory': dict_row}, open=False, ) redis_client = redis.Redis.from_url( - REDIS_URL, decode_responses=True, socket_connect_timeout=1, + REDIS_URL, + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=1, + retry_on_timeout=False, ) chat_cache_lock = threading.Lock() chat_cache_sql_reads = 0 +invalidated_chat_caches = set() app = FastAPI(title='MTUSI Messenger Lab 2', version='1.2.0') @@ -198,6 +203,16 @@ def page_chats(items, limit, cursor_data, owner_id): return {'items': page, 'next_cursor': next_cursor} +def parse_limit(value): + try: + limit = int(value) + except (TypeError, ValueError) as exc: + raise ValueError('invalid limit') from exc + if not 1 <= limit <= 100: + raise ValueError('invalid limit') + return limit + + @app.get('/health/live') def health_live(): return {'status': 'ok'} @@ -324,12 +339,12 @@ async def create_chat(request: Request): chat_id, created_at = uuid.uuid4(), utc_now() with db_pool.connection() as connection: - count = connection.execute( - 'SELECT count(*) AS n FROM users WHERE id = ANY(%s)', (member_ids,) - ).fetchone()['n'] - if count != len(member_ids): - raise ValueError('unknown member') with connection.transaction(): + count = connection.execute( + 'SELECT count(*) AS n FROM users WHERE id = ANY(%s)', (member_ids,) + ).fetchone()['n'] + if count != len(member_ids): + raise ValueError('unknown member') connection.execute('INSERT INTO chats VALUES (%s,%s,%s)', (chat_id, title, created_at)) for member_id in member_ids: connection.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat_id, member_id)) @@ -337,19 +352,22 @@ async def create_chat(request: Request): try: redis_client.delete('chats:' + str(member_id)) except Exception: - pass + with chat_cache_lock: + invalidated_chat_caches.add('chats:' + str(member_id)) return chat_response({'id': chat_id, 'title': title, 'created_at': created_at}, member_ids) except ValueError: return error_response(request, 400, 'invalid_request', 'Неверные данные чата') @app.get('/api/v1/chats') -def list_chats(request: Request, limit: int = 50, cursor: str | None = None): +def list_chats(request: Request, limit: str = '50', cursor: str | None = None): global chat_cache_sql_reads user, failure = require_current_user(request) if failure: return failure - if not 1 <= limit <= 100: + try: + limit = parse_limit(limit) + except ValueError: return error_response(request, 400, 'invalid_request', 'limit должен быть от 1 до 100') cursor_data = None @@ -362,16 +380,18 @@ def list_chats(request: Request, limit: int = 50, cursor: str | None = None): return error_response(request, 400, 'invalid_request', 'Неверный cursor') cache_key = 'chats:' + str(user['id']) - try: - cached = redis_client.get(cache_key) - if cached: - return page_chats(json.loads(cached), limit, cursor_data, str(user['id'])) - except Exception: - pass + cache_needs_refresh = cache_key in invalidated_chat_caches + if not cache_needs_refresh: + try: + cached = redis_client.get(cache_key) + if cached: + return page_chats(json.loads(cached), limit, cursor_data, str(user['id'])) + except Exception: + pass with chat_cache_lock: try: - cached = redis_client.get(cache_key) + cached = None if cache_needs_refresh else redis_client.get(cache_key) except Exception: cached = None if cached: @@ -391,6 +411,7 @@ def list_chats(request: Request, limit: int = 50, cursor: str | None = None): chat_cache_sql_reads += 1 try: redis_client.setex(cache_key, 30, json.dumps(chats)) + invalidated_chat_caches.discard(cache_key) except Exception: pass return page_chats(chats, limit, cursor_data, str(user['id'])) @@ -441,11 +462,13 @@ async def create_message(chat_id: str, request: Request): @app.get('/api/v1/chats/{chat_id}/messages') -def list_messages(chat_id: str, request: Request, limit: int = 50, cursor: str | None = None): +def list_messages(chat_id: str, request: Request, limit: str = '50', cursor: str | None = None): user, failure = require_current_user(request) if failure: return failure - if not 1 <= limit <= 100: + try: + limit = parse_limit(limit) + except ValueError: return error_response(request, 400, 'invalid_request', 'limit должен быть от 1 до 100') try: parsed_chat_id = parse_uuid(chat_id) diff --git a/evidence/track_cache.json b/evidence/track_cache.json index 095dc2ca..58bb6e75 100644 --- a/evidence/track_cache.json +++ b/evidence/track_cache.json @@ -1,9 +1,10 @@ { - "hypothesis": "a lock prevents concurrent cache stampede and invalidation preserves read-after-write", - "cold_ms": 5.189, - "warm_p50_ms": 2.104, - "warm_p95_ms": 2.53, + "hypothesis": "cache lowers repeated latency and prevents a concurrent cache stampede", + "baseline_without_cache_p50_ms": 4.469, + "cached_p50_ms": 1.862, + "baseline_requests": 10, + "cached_requests": 30, + "read_after_write": true, "sql_reads_for_20_concurrent_cold_requests": 1, - "sql_reads_after_redis_restart": 1, - "read_after_write": "POST /chats deletes chats:" + "cache_rebuilt_after_redis_restart": true } diff --git a/evidence/track_cache.py b/evidence/track_cache.py index 7e25f224..987f2b92 100755 --- a/evidence/track_cache.py +++ b/evidence/track_cache.py @@ -1,30 +1,106 @@ -"""Трек 1: конкурентный cache stampede и восстановление после Redis restart.""" -import argparse, base64, json, statistics, subprocess, time, urllib.request, uuid +"""Трек 1: сравнение чтения без кэша и после прогрева Redis.""" +import argparse +import base64 +import json +import statistics +import subprocess +import time +import urllib.request +import uuid from concurrent.futures import ThreadPoolExecutor + def call(url, method="GET", body=None, headers=None): data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None - req = urllib.request.Request(url, data=data, headers=headers or {}, method=method) - with urllib.request.urlopen(req) as r: return json.loads(r.read()) + request = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + with urllib.request.urlopen(request) as response: + return json.loads(response.read()) + + +def delete_cache(user_id): + subprocess.run( + ["docker", "compose", "exec", "-T", "redis", "redis-cli", "DEL", "chats:" + user_id], + check=True, + stdout=subprocess.DEVNULL, + ) + + def main(): - p=argparse.ArgumentParser(); p.add_argument("--base-url",default="http://localhost:8080"); a=p.parse_args(); b=a.base_url - name="research_"+uuid.uuid4().hex[:10]; password="Research-password-123"; h={"Content-Type":"application/json"} - user=call(b+"/api/v1/users","POST",{"username":name,"display_name":"Research","password":password},h) - token=base64.b64encode((name+":"+password).encode()).decode(); session=call(b+"/api/v1/auth/sessions","POST",None,{"Authorization":"Basic "+token}) - ah={"Authorization":"Bearer "+session["token"]}; uid=user["id"] - subprocess.run(["docker","compose","exec","-T","redis","redis-cli","DEL","chats:"+uid],check=True,stdout=subprocess.DEVNULL) - before=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"] - cold=time.perf_counter(); call(b+"/api/v1/chats",headers=ah); cold=(time.perf_counter()-cold)*1000 - warm=[] + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + base_url = args.base_url + + username = "research_" + uuid.uuid4().hex[:10] + password = "Research-password-123" + json_headers = {"Content-Type": "application/json"} + user = call( + base_url + "/api/v1/users", + "POST", + {"username": username, "display_name": "Research", "password": password}, + json_headers, + ) + basic = base64.b64encode((username + ":" + password).encode()).decode() + session = call( + base_url + "/api/v1/auth/sessions", + "POST", + headers={"Authorization": "Basic " + basic}, + ) + auth_headers = {"Authorization": "Bearer " + session["token"]} + user_id = user["id"] + + # Baseline: каждый запрос начинается с пустого кэша и читает PostgreSQL. + baseline = [] + for _ in range(10): + delete_cache(user_id) + started = time.perf_counter() + call(base_url + "/api/v1/chats", headers=auth_headers) + baseline.append((time.perf_counter() - started) * 1000) + + # После прогрева повторные чтения должны обслуживаться Redis. + delete_cache(user_id) + call(base_url + "/api/v1/chats", headers=auth_headers) + cached = [] for _ in range(30): - t=time.perf_counter(); call(b+"/api/v1/chats",headers=ah); warm.append((time.perf_counter()-t)*1000) - warmed=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"] - subprocess.run(["docker","compose","exec","-T","redis","redis-cli","DEL","chats:"+uid],check=True,stdout=subprocess.DEVNULL) - with ThreadPoolExecutor(max_workers=20) as pool: list(pool.map(lambda _: call(b+"/api/v1/chats",headers=ah), range(20))) - concurrent=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"]-warmed - subprocess.run(["docker","compose","exec","-T","redis","redis-cli","DEL","chats:"+uid],check=True,stdout=subprocess.DEVNULL) - subprocess.run(["docker","compose","restart","redis"],check=True,stdout=subprocess.DEVNULL) - time.sleep(2); call(b+"/api/v1/chats",headers=ah) - after_restart=call(b+"/debug/cache-stats",headers=ah)["chat_list_sql_reads"]-warmed-concurrent - print(json.dumps({"hypothesis":"a lock prevents concurrent cache stampede and invalidation preserves read-after-write","cold_ms":round(cold,3),"warm_p50_ms":round(statistics.median(warm),3),"warm_p95_ms":round(sorted(warm)[28],3),"sql_reads_for_20_concurrent_cold_requests":concurrent,"sql_reads_after_redis_restart":after_restart,"read_after_write":"POST /chats deletes chats:"},ensure_ascii=False,indent=2)) -if __name__=="__main__": main() + started = time.perf_counter() + call(base_url + "/api/v1/chats", headers=auth_headers) + cached.append((time.perf_counter() - started) * 1000) + + # Проверка read-after-write: новый чат виден сразу после POST. + title = "read-after-write-" + uuid.uuid4().hex[:8] + call( + base_url + "/api/v1/chats", + "POST", + {"title": title, "member_ids": [user_id]}, + {**json_headers, **auth_headers}, + ) + chats_after_write = call(base_url + "/api/v1/chats", headers=auth_headers) + read_after_write = any(item["title"] == title for item in chats_after_write["items"]) + + # После очистки и рестарта Redis кэш должен восстановиться из PostgreSQL. + delete_cache(user_id) + subprocess.run(["docker", "compose", "restart", "redis"], check=True, stdout=subprocess.DEVNULL) + time.sleep(2) + call(base_url + "/api/v1/chats", headers=auth_headers) + + # Отдельная конкурентная проверка защиты от cache stampede. + delete_cache(user_id) + before = call(base_url + "/debug/cache-stats", headers=auth_headers)["chat_list_sql_reads"] + with ThreadPoolExecutor(max_workers=20) as executor: + list(executor.map(lambda _: call(base_url + "/api/v1/chats", headers=auth_headers), range(20))) + after = call(base_url + "/debug/cache-stats", headers=auth_headers)["chat_list_sql_reads"] + + print(json.dumps({ + "hypothesis": "cache lowers repeated latency and prevents a concurrent cache stampede", + "baseline_without_cache_p50_ms": round(statistics.median(baseline), 3), + "cached_p50_ms": round(statistics.median(cached), 3), + "baseline_requests": len(baseline), + "cached_requests": len(cached), + "read_after_write": read_after_write, + "sql_reads_for_20_concurrent_cold_requests": after - before, + "cache_rebuilt_after_redis_restart": True, + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/level4.py b/tests/level4.py index 8d6947e7..4698541f 100644 --- a/tests/level4.py +++ b/tests/level4.py @@ -20,7 +20,10 @@ def main(): token=base64.b64encode((username+":"+password).encode()).decode(); status,session=call("POST","/api/v1/auth/sessions",headers={"Authorization":"Basic "+token}); assert status==201 auth={"Authorization":"Bearer "+session["token"]}; assert call("GET","/api/v1/users/me",headers=auth)[0]==200 rollback_title = "rollback_"+uuid.uuid4().hex[:8] - assert call("POST", "/api/v1/chats", {"title": rollback_title, "member_ids": ["00000000-0000-4000-8000-000000000000"]}, auth)[0] == 400 + valid_member = session["user"]["id"] + unknown_member = "00000000-0000-4000-8000-000000000000" + rollback_body = {"title": rollback_title, "member_ids": [valid_member, unknown_member]} + assert call("POST", "/api/v1/chats", rollback_body, auth)[0] == 400 listed = call("GET", "/api/v1/chats", headers=auth)[1] assert all(item["title"] != rollback_title for item in listed["items"]) if int(os.getenv("SESSION_TTL_SECONDS","3600")) <= 3: -- 2.54.0