diff --git a/db.py b/db.py index 1b82471..5f39a3b 100644 --- a/db.py +++ b/db.py @@ -26,7 +26,6 @@ pool = ConnectionPool( "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", }, - check=ConnectionPool.check_connection, open=False, ) @@ -34,16 +33,29 @@ pool = ConnectionPool( @contextmanager def connection(wait_seconds: float | None = None): - """Соединение из пула. В конце блока COMMIT, при исключении ROLLBACK. - - Если база недоступна, клиент получает 503 без подробностей. - """ try: with pool.connection(timeout=wait_seconds) as conn: yield conn except (psycopg.OperationalError, PoolTimeout) as exc: log.warning("PostgreSQL is unavailable: %s", type(exc).__name__) raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} def ping() -> None: @@ -51,8 +63,7 @@ def ping() -> None: conn.execute("SELECT 1") - -MIGRATIONS_LOCK = 20260202 +MIGRATIONS_LOCK = 20260202 def run_migrations() -> list[str]: @@ -100,26 +111,25 @@ USER_COLUMNS = "id, username, display_name, created_at" def create_user(username: str, display_name: str, password_hash: str) -> dict: try: with connection() as conn: - return conn.execute( + return from_db(conn.execute( f"""INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", - (uuid4(), username, display_name, password_hash), - ).fetchone() + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) except pg_errors.UniqueViolation: raise conflict("Username already exists") from None def get_user(user_id: UUID) -> dict | None: with connection() as conn: - return conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone() + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) def get_user_with_password(username: str) -> dict | None: with connection() as conn: - return conn.execute( + return from_db(conn.execute( f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) - ).fetchone() - + ).fetchone()) CHAT_COLUMNS = """c.id, c.title, c.created_at, @@ -134,7 +144,7 @@ def create_chat(title: str, member_ids: list[UUID]) -> dict: with conn.transaction(): row = conn.execute( "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", - (chat_id, title), + (chat_id, to_db(title)), ).fetchone() with conn.cursor() as cur: cur.executemany( @@ -143,7 +153,7 @@ def create_chat(title: str, member_ids: list[UUID]) -> dict: ) except pg_errors.ForeignKeyViolation: raise bad_request("Unknown member") from None - return {**row, "member_ids": members} + return {**from_db(row), "member_ids": members} def get_chat(chat_id: UUID, user_id: UUID) -> dict: @@ -156,7 +166,7 @@ def get_chat(chat_id: UUID, user_id: UUID) -> dict: ).fetchone() if chat is None: raise not_found("Chat not found") - return chat + return from_db(chat) def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: @@ -166,7 +176,7 @@ def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) - where = "WHERE (c.created_at, c.id) > (%s, %s)" params += list(after) with connection() as conn: - return conn.execute( + rows = conn.execute( f"""SELECT {CHAT_COLUMNS} FROM chats c JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s {where} @@ -174,6 +184,7 @@ def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) - LIMIT %s""", (*params, limit + 1), ).fetchall() + return [from_db(row) for row in rows] @@ -183,11 +194,11 @@ MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" def add_message(chat_id: UUID, sender_id: UUID, text: str) -> dict: try: with connection() as conn: - return conn.execute( + return from_db(conn.execute( f"""INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s, %s, %s, %s) RETURNING {MESSAGE_COLUMNS}""", - (uuid4(), chat_id, sender_id, text), - ).fetchone() + (uuid4(), chat_id, sender_id, to_db(text)), + ).fetchone()) except pg_errors.ForeignKeyViolation: raise not_found("Chat not found") from None @@ -204,10 +215,11 @@ def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | N ).fetchone() if member is None: raise not_found("Chat not found") - return conn.execute( + rows = conn.execute( f"""SELECT {MESSAGE_COLUMNS} FROM messages WHERE {where} ORDER BY created_at, id LIMIT %s""", (*params, limit + 1), ).fetchall() + return [from_db(row) for row in rows] diff --git a/evidence/db.txt b/evidence/db.txt index 8e99167..9bc82d3 100644 Binary files a/evidence/db.txt and b/evidence/db.txt differ diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt index d4cc7fb..d01e5ea 100644 --- a/evidence/explain-history.txt +++ b/evidence/explain-history.txt @@ -1,7 +1,7 @@ === Data volume messages_total | messages_in_big_chat ----------------+---------------------- - 20116 | 15000 + 20250 | 15000 === History index @@ -13,58 +13,58 @@ === 1. First page: limit=50, the app asks for 51 rows to know if there is a next page QUERY PLAN ------------------------------------------------------------------------------------------------ - Limit (actual time=0.007..0.012 rows=51 loops=1) + Limit (actual time=0.005..0.011 rows=51 loops=1) Buffers: shared hit=4 - -> Index Scan using messages_history on messages (actual time=0.007..0.009 rows=51 loops=1) - Index Cond: (chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) + -> Index Scan using messages_history on messages (actual time=0.004..0.008 rows=51 loops=1) + Index Cond: (chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) Buffers: shared hit=4 Planning: - Buffers: shared hit=8 - Planning Time: 0.155 ms - Execution Time: 0.028 ms + Buffers: shared hit=6 + Planning Time: 0.055 ms + Execution Time: 0.020 ms === 2. Page after the 10000th message by keyset cursor (this is what the app does) QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Limit (actual time=0.006..0.012 rows=51 loops=1) - Buffers: shared hit=6 - -> Index Scan using messages_history on messages (actual time=0.006..0.010 rows=51 loops=1) - Index Cond: ((chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'd867a7e3-2b56-475a-81c9-19729e8336c2'::uuid))) - Buffers: shared hit=6 + Limit (actual time=0.006..0.011 rows=51 loops=1) + Buffers: shared hit=4 + -> Index Scan using messages_history on messages (actual time=0.006..0.008 rows=51 loops=1) + Index Cond: ((chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '3b67880a-c02c-48ac-aec2-1a0331e99fee'::uuid))) + Buffers: shared hit=4 Planning Time: 0.038 ms - Execution Time: 0.031 ms + Execution Time: 0.028 ms === 3. The same page with OFFSET 10000 (the app does NOT do this) QUERY PLAN --------------------------------------------------------------------------------------------------- - Limit (actual time=0.716..0.721 rows=51 loops=1) - Buffers: shared hit=257 - -> Index Scan using messages_history on messages (actual time=0.008..0.521 rows=10051 loops=1) - Index Cond: (chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) - Buffers: shared hit=257 - Planning Time: 0.028 ms + Limit (actual time=0.747..0.752 rows=51 loops=1) + Buffers: shared hit=199 + -> Index Scan using messages_history on messages (actual time=0.006..0.566 rows=10051 loops=1) + Index Cond: (chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) + Buffers: shared hit=199 + Planning Time: 0.033 ms Execution Time: 0.765 ms === 4. Query 2 without the index (index is dropped only inside this transaction) QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Limit (actual time=0.862..0.865 rows=51 loops=1) - Buffers: shared hit=255 - -> Sort (actual time=0.861..0.863 rows=51 loops=1) + Limit (actual time=0.895..0.898 rows=51 loops=1) + Buffers: shared hit=251 + -> Sort (actual time=0.895..0.896 rows=51 loops=1) Sort Key: created_at, id Sort Method: top-N heapsort Memory: 31kB - Buffers: shared hit=255 - -> Seq Scan on messages (actual time=0.289..0.686 rows=5000 loops=1) - Filter: ((chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'd867a7e3-2b56-475a-81c9-19729e8336c2'::uuid))) - Rows Removed by Filter: 15116 - Buffers: shared hit=255 + Buffers: shared hit=251 + -> Seq Scan on messages (actual time=0.320..0.719 rows=5000 loops=1) + Filter: ((chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '3b67880a-c02c-48ac-aec2-1a0331e99fee'::uuid))) + Rows Removed by Filter: 15250 + Buffers: shared hit=251 Planning: Buffers: shared hit=4 - Planning Time: 0.051 ms - Execution Time: 0.884 ms + Planning Time: 0.043 ms + Execution Time: 0.912 ms === ROLLBACK: test data removed, index is back diff --git a/evidence/persistence.txt b/evidence/persistence.txt index 2c5670c..81eb87e 100644 Binary files a/evidence/persistence.txt and b/evidence/persistence.txt differ diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt index 1440c6b..a46dd3e 100644 Binary files a/evidence/student-tests.txt and b/evidence/student-tests.txt differ diff --git a/main.py b/main.py index 1f3cf57..d235411 100644 --- a/main.py +++ b/main.py @@ -1,16 +1,8 @@ -"""Мессенджер, lab2: HTTP API по contracts/openapi.json. - -Что изменилось с lab1: -- данные (users, chats, messages) теперь в PostgreSQL, а не в словарях: см. db.py; -- регистрация принимает password, вход — POST /auth/sessions с Basic, выход — DELETE; -- текущий пользователь определяется только по Bearer-токену (сессии в Redis, см. auth.py), - заголовок X-User-Id больше ничего не значит. -Модели, формат ошибок, X-Request-Id и курсоры остались из lab1. -""" import base64 import binascii import json import logging +import re import time import uuid from contextlib import asynccontextmanager @@ -49,6 +41,8 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan) +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") + class CreateUser(BaseModel): model_config = ConfigDict(extra="forbid") @@ -209,7 +203,7 @@ async def request_context(request: Request, call_next): auth.check_bearer, request.headers.get("Authorization") ) response = await call_next(request) - except ApiError as exc: + except ApiError as exc: response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) except Exception: log.exception("Unhandled error, request_id=%s", request_id) @@ -297,7 +291,6 @@ def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID return created_at, object_id - @app.get("/health/live") def health_live(): return {"status": "ok"} @@ -310,7 +303,6 @@ def health_ready(): return {"status": "ok"} - @app.post("/api/v1/users", response_model=User, status_code=201) def create_user(user_data: CreateUser): password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется @@ -328,7 +320,7 @@ def create_session( username, password = credentials found = None - if len(password) <= auth.PASSWORD_MAX_LENGTH: + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: found = db.get_user_with_password(username) if not auth.password_matches(found["password_hash"] if found else None, password): raise auth.login_failed() diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 0000000..73f0c84 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/student-tests/explain.sql b/student-tests/explain.sql index a5841c2..39e9a56 100644 --- a/student-tests/explain.sql +++ b/student-tests/explain.sql @@ -1,31 +1,24 @@ --- Обоснование индекса истории messages_history (migrations/001_init.sql). --- Запуск: python student-tests/explain.py (результат -> evidence/explain-history.txt) --- --- Всё в одной транзакции, в конце ROLLBACK: тестовые данные не остаются в базе, --- скрипт можно запускать сколько угодно раз. \pset footer off BEGIN; --- Тестовые данные: один большой чат на 15 000 сообщений и 50 маленьких по 100, --- всего 20 000 сообщений. Чужие сообщения в таблице есть, как в настоящей базе. CREATE TEMP TABLE seed_chats AS SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size FROM generate_series(0, 50) AS n; CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; INSERT INTO users (id, username, display_name, password_hash) - SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), 'Explain', 'not-a-real-hash' FROM seed_user; -INSERT INTO chats (id, title) SELECT id, 'explain ' || n FROM seed_chats; + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; INSERT INTO messages (id, chat_id, sender_id, text, created_at) - SELECT gen_random_uuid(), c.id, u.id, 'message ' || g, + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; ANALYZE messages; SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset --- Курсор глубоко в истории: ключ (created_at, id) 10 000-го сообщения большого чата. SELECT created_at AS cursor_time, id AS cursor_id FROM messages WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset diff --git a/student-tests/test_api.py b/student-tests/test_api.py index 382aeeb..0440df6 100644 --- a/student-tests/test_api.py +++ b/student-tests/test_api.py @@ -1,10 +1,3 @@ -"""Собственные тесты API (lab1 + lab2). Запуск одной командой при запущенном стенде: - - python student-tests/test_api.py - -Тесты lab1 переведены с X-User-Id на пароль и Bearer-токен. Добавлены тесты lab2: -вход по Basic, неверный пароль, logout, Bearer на всех защищённых запросах. -""" import base64 import datetime as dt import uuid @@ -16,7 +9,6 @@ def parse_time(value): return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) -# ---------- lab1 (перенесены, теперь с токенами) ---------- def test_duplicate_username(): body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} @@ -79,7 +71,6 @@ def test_pagination(): for i in range(3): send(alice, other["id"], f"Other {i}") first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json - # Курсор от другого чата и испорченный курсор — 400. call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) print("PASS: pagination works across multiple pages, foreign cursor returns 400") @@ -110,7 +101,23 @@ def test_unicode(): print("PASS: unicode text is stored as is") -# ---------- lab2: пароль, вход, Bearer, logout ---------- +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + def test_user_has_no_password(): body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} @@ -155,7 +162,6 @@ def test_wrong_password(): wrong = login(user["username"], "Wrong-password-000", expected=401) unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") - # Одинаковый ответ: по нему нельзя узнать, существует ли такой username. check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") print("PASS: wrong password and unknown user both return the same 401") @@ -196,7 +202,6 @@ def test_x_user_id_ignored(): def test_auth_before_body(): alice = new_user("alice") chat = new_chat(alice) - # Без токена всегда 401, даже если тело неверное или чат чужой: сначала «кто ты». call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) print("PASS: 401 comes before 400 and 404") @@ -225,6 +230,7 @@ if __name__ == "__main__": test_pagination, test_chat_list_pagination, test_unicode, + test_nul_character_is_stored, test_user_has_no_password, test_password_rules, test_login_gives_token, diff --git a/student-tests/test_db.py b/student-tests/test_db.py index 29c4034..3f9c83b 100644 --- a/student-tests/test_db.py +++ b/student-tests/test_db.py @@ -1,14 +1,3 @@ -"""Проверки на уровне базы данных. Запуск при запущенном стенде: - - python student-tests/test_db.py - -SQL выполняется в контейнере postgres через `docker compose exec postgres psql`. -1. Откат составной операции: чат с несуществующим участником -> 400, и в базе - не остаётся ни чата, ни участников, хотя строка чата успела вставиться. -2. База сама не даёт записать сообщение от не-участника (внешний ключ). -3. Пароль хранится как хеш Argon2id, а не как текст или быстрый хеш. -4. Миграции: после перезапуска API ничего не применяется повторно. -""" import time import uuid @@ -26,14 +15,12 @@ def test_create_chat_rolls_back(): title = "rollback-" + uuid.uuid4().hex[:8] inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) - # Второй участник не существует: INSERT чата пройдёт, INSERT участника упадёт на внешнем ключе. call("POST", "/api/v1/chats", 400, token=alice["token"], body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") - # Статистика PostgreSQL считает и отменённые вставки; она обновляется с задержкой до ~1 с. inserted_after = inserted_before deadline = time.monotonic() + 15 while inserted_after == inserted_before and time.monotonic() < deadline: @@ -48,7 +35,7 @@ def test_create_chat_rolls_back(): def test_message_needs_membership_in_db(): code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " - f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', 'direct insert')") + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") print("PASS: the database itself rejects a message from a non-member")