From e17d883c158da3e035202b6a4ae10bb197cf5ef9 Mon Sep 17 00:00:00 2001 From: kalibr456 <145665723+kalibr456@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:30:42 +0300 Subject: [PATCH] =?UTF-8?q?=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5:=20U+0000=20=D0=B2=20=D1=81=D1=82=D1=80?= =?UTF-8?q?=D0=BE=D0=BA=D0=B0=D1=85=20=D1=81=D0=BE=D1=85=D1=80=D0=B0=D0=BD?= =?UTF-8?q?=D1=8F=D0=B5=D1=82=D1=81=D1=8F=20(bytea),=20DataError=20->=2040?= =?UTF-8?q?0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db.py | 56 +++++++++++++++++------------ evidence/db.txt | Bin 1664 -> 1754 bytes evidence/explain-history.txt | 58 +++++++++++++++---------------- evidence/persistence.txt | Bin 2690 -> 1068 bytes evidence/student-tests.txt | Bin 1764 -> 1878 bytes main.py | 18 +++------- migrations/002_text_as_bytes.sql | 11 ++++++ student-tests/explain.sql | 15 +++----- student-tests/test_api.py | 30 +++++++++------- student-tests/test_db.py | 15 +------- 10 files changed, 102 insertions(+), 101 deletions(-) create mode 100644 migrations/002_text_as_bytes.sql diff --git a/db.py b/db.py index 1b82471..5f39a3b 100644 --- a/db.py +++ b/db.py @@ -26,7 +26,6 @@ pool = ConnectionPool( "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", }, - check=ConnectionPool.check_connection, open=False, ) @@ -34,16 +33,29 @@ pool = ConnectionPool( @contextmanager def connection(wait_seconds: float | None = None): - """Соединение из пула. В конце блока COMMIT, при исключении ROLLBACK. - - Если база недоступна, клиент получает 503 без подробностей. - """ try: with pool.connection(timeout=wait_seconds) as conn: yield conn except (psycopg.OperationalError, PoolTimeout) as exc: log.warning("PostgreSQL is unavailable: %s", type(exc).__name__) raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} def ping() -> None: @@ -51,8 +63,7 @@ def ping() -> None: conn.execute("SELECT 1") - -MIGRATIONS_LOCK = 20260202 +MIGRATIONS_LOCK = 20260202 def run_migrations() -> list[str]: @@ -100,26 +111,25 @@ USER_COLUMNS = "id, username, display_name, created_at" def create_user(username: str, display_name: str, password_hash: str) -> dict: try: with connection() as conn: - return conn.execute( + return from_db(conn.execute( f"""INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", - (uuid4(), username, display_name, password_hash), - ).fetchone() + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) except pg_errors.UniqueViolation: raise conflict("Username already exists") from None def get_user(user_id: UUID) -> dict | None: with connection() as conn: - return conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone() + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) def get_user_with_password(username: str) -> dict | None: with connection() as conn: - return conn.execute( + return from_db(conn.execute( f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) - ).fetchone() - + ).fetchone()) CHAT_COLUMNS = """c.id, c.title, c.created_at, @@ -134,7 +144,7 @@ def create_chat(title: str, member_ids: list[UUID]) -> dict: with conn.transaction(): row = conn.execute( "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", - (chat_id, title), + (chat_id, to_db(title)), ).fetchone() with conn.cursor() as cur: cur.executemany( @@ -143,7 +153,7 @@ def create_chat(title: str, member_ids: list[UUID]) -> dict: ) except pg_errors.ForeignKeyViolation: raise bad_request("Unknown member") from None - return {**row, "member_ids": members} + return {**from_db(row), "member_ids": members} def get_chat(chat_id: UUID, user_id: UUID) -> dict: @@ -156,7 +166,7 @@ def get_chat(chat_id: UUID, user_id: UUID) -> dict: ).fetchone() if chat is None: raise not_found("Chat not found") - return chat + return from_db(chat) def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: @@ -166,7 +176,7 @@ def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) - where = "WHERE (c.created_at, c.id) > (%s, %s)" params += list(after) with connection() as conn: - return conn.execute( + rows = conn.execute( f"""SELECT {CHAT_COLUMNS} FROM chats c JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s {where} @@ -174,6 +184,7 @@ def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) - LIMIT %s""", (*params, limit + 1), ).fetchall() + return [from_db(row) for row in rows] @@ -183,11 +194,11 @@ MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" def add_message(chat_id: UUID, sender_id: UUID, text: str) -> dict: try: with connection() as conn: - return conn.execute( + return from_db(conn.execute( f"""INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s, %s, %s, %s) RETURNING {MESSAGE_COLUMNS}""", - (uuid4(), chat_id, sender_id, text), - ).fetchone() + (uuid4(), chat_id, sender_id, to_db(text)), + ).fetchone()) except pg_errors.ForeignKeyViolation: raise not_found("Chat not found") from None @@ -204,10 +215,11 @@ def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | N ).fetchone() if member is None: raise not_found("Chat not found") - return conn.execute( + rows = conn.execute( f"""SELECT {MESSAGE_COLUMNS} FROM messages WHERE {where} ORDER BY created_at, id LIMIT %s""", (*params, limit + 1), ).fetchall() + return [from_db(row) for row in rows] diff --git a/evidence/db.txt b/evidence/db.txt index 8e991673ee5b852c33c2497eee7febeb4dfd6147..9bc82d38ea1deb9b77a60a3e029dcb00f7df72af 100644 GIT binary patch delta 109 zcmZqRy~R7>nxG|v1%m>EE`uGAG-NPfFxdFioJCm }aFh-WBaNM)!1lJN|Q48=e) ViJ=lKRy<{9 diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt index d4cc7fb..d01e5ea 100644 --- a/evidence/explain-history.txt +++ b/evidence/explain-history.txt @@ -1,7 +1,7 @@ === Data volume messages_total | messages_in_big_chat ----------------+---------------------- - 20116 | 15000 + 20250 | 15000 === History index @@ -13,58 +13,58 @@ === 1. First page: limit=50, the app asks for 51 rows to know if there is a next page QUERY PLAN ------------------------------------------------------------------------------------------------ - Limit (actual time=0.007..0.012 rows=51 loops=1) + Limit (actual time=0.005..0.011 rows=51 loops=1) Buffers: shared hit=4 - -> Index Scan using messages_history on messages (actual time=0.007..0.009 rows=51 loops=1) - Index Cond: (chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) + -> Index Scan using messages_history on messages (actual time=0.004..0.008 rows=51 loops=1) + Index Cond: (chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) Buffers: shared hit=4 Planning: - Buffers: shared hit=8 - Planning Time: 0.155 ms - Execution Time: 0.028 ms + Buffers: shared hit=6 + Planning Time: 0.055 ms + Execution Time: 0.020 ms === 2. Page after the 10000th message by keyset cursor (this is what the app does) QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Limit (actual time=0.006..0.012 rows=51 loops=1) - Buffers: shared hit=6 - -> Index Scan using messages_history on messages (actual time=0.006..0.010 rows=51 loops=1) - Index Cond: ((chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'd867a7e3-2b56-475a-81c9-19729e8336c2'::uuid))) - Buffers: shared hit=6 + Limit (actual time=0.006..0.011 rows=51 loops=1) + Buffers: shared hit=4 + -> Index Scan using messages_history on messages (actual time=0.006..0.008 rows=51 loops=1) + Index Cond: ((chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '3b67880a-c02c-48ac-aec2-1a0331e99fee'::uuid))) + Buffers: shared hit=4 Planning Time: 0.038 ms - Execution Time: 0.031 ms + Execution Time: 0.028 ms === 3. The same page with OFFSET 10000 (the app does NOT do this) QUERY PLAN --------------------------------------------------------------------------------------------------- - Limit (actual time=0.716..0.721 rows=51 loops=1) - Buffers: shared hit=257 - -> Index Scan using messages_history on messages (actual time=0.008..0.521 rows=10051 loops=1) - Index Cond: (chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) - Buffers: shared hit=257 - Planning Time: 0.028 ms + Limit (actual time=0.747..0.752 rows=51 loops=1) + Buffers: shared hit=199 + -> Index Scan using messages_history on messages (actual time=0.006..0.566 rows=10051 loops=1) + Index Cond: (chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) + Buffers: shared hit=199 + Planning Time: 0.033 ms Execution Time: 0.765 ms === 4. Query 2 without the index (index is dropped only inside this transaction) QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Limit (actual time=0.862..0.865 rows=51 loops=1) - Buffers: shared hit=255 - -> Sort (actual time=0.861..0.863 rows=51 loops=1) + Limit (actual time=0.895..0.898 rows=51 loops=1) + Buffers: shared hit=251 + -> Sort (actual time=0.895..0.896 rows=51 loops=1) Sort Key: created_at, id Sort Method: top-N heapsort Memory: 31kB - Buffers: shared hit=255 - -> Seq Scan on messages (actual time=0.289..0.686 rows=5000 loops=1) - Filter: ((chat_id = '6a75871f-4837-4b3c-b7c8-1c36de87da98'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'd867a7e3-2b56-475a-81c9-19729e8336c2'::uuid))) - Rows Removed by Filter: 15116 - Buffers: shared hit=255 + Buffers: shared hit=251 + -> Seq Scan on messages (actual time=0.320..0.719 rows=5000 loops=1) + Filter: ((chat_id = 'edccb516-a15c-427e-a8d3-9b1b2d9ea77c'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '3b67880a-c02c-48ac-aec2-1a0331e99fee'::uuid))) + Rows Removed by Filter: 15250 + Buffers: shared hit=251 Planning: Buffers: shared hit=4 - Planning Time: 0.051 ms - Execution Time: 0.884 ms + Planning Time: 0.043 ms + Execution Time: 0.912 ms === ROLLBACK: test data removed, index is back diff --git a/evidence/persistence.txt b/evidence/persistence.txt index 2c5670c7752fa51e2f9b0988e37f6d81648c2e3f..81eb87ea425c8b6a46273147f31b4321ff836d5f 100644 GIT binary patch delta 183 zcmZn?UBfZKMKPHniNTD)oWYpElpz@iO@V9^1|tRqhERqAAZavNmQhjI3dkyEC}Bus zC<2n140#Oc6FV*WLCSK$+)|)$>SRGKcPwJko81_z7$>`MI*H@3ZL&X?@#G7fo0N(e dQh|2lFo5hc075GuwqVc$y3Pd1;$`4s004utA!h&p literal 2690 zcmcJRUr!T36vgkuKAHF(<`JSL6%@2aBVq}>pcc>v<3puO+1Njuc0u~$M^}I6PMa-| z25h#ObocI^nKS2}IXm6Ie~ztbE&F1Jwnr4WezYSyv0XAp-WP~I&~xnB6FVh(Y>%vM zxn<;ctZuLD8Q15wZ9ll55pB|Q%7|buXu-yok{{Z@I>f=&?1D_&dR7okY~3+)MvTdH zZA7oeb-=E(#*X9EdEWL_mO&S02hPG3Yhi7{ybarz#3{LDaGQ3`+%vm`c}9g@P*QR^ z)n()}@`0z6s>Z&4K|G)*C4WK9L;Gp3=i)#{Xt`gd-X(E9RYh8M7+o+b!2XJR=9wNn zZLnr6qUx%<0As^7{>0uM(;CcO#A&vY7++0Si@SFY_Q?9vT!+;Peqy1Wjde@mQVDSw z(=u2N;1KM9NENFtp^8PIFLhMyPRxxQeaL#kz497oy2V;Lj64HLJcCyx2WuU%k_xd$ ze8oYtyPm7DH*MUyX*K4~d{M&b&q?+5sZhRFmD@fPd4!M<+XqhbT>0wz(@br8SVcYDgc#mv61o^s+``+JeQ zgS6u9y7tkqN1NDZBAp;5*CzNQulLXuj&;eSnfqAY=+mNG%h7R-c>RC>+&;AyyJ#7A z#&fn>%teg9PTg4gei_Dh-=|gK9^jnvKcwpYTP^ykG^qynrt7M~T;#fHbiV7lYH)8V gaTjy-Ur^hek@25g=V7(qPY=f~^}YY=>EWpV0OzQ`J^%m! diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt index 1440c6b6db1ccbaa53a0a393409f2f9c8eb94970..a46dd3e6615881f783d1d00c49fb503fb9c25727 100644 GIT binary patch delta 100 zcmaFDdyQ|y0j9uWh7yJ%hD?S$hIEEv1_g$4AiIPi1IP+x&}J}zLIs9IpsG}$C`f%i gn4JRTB?9$ evidence/explain-history.txt) --- --- Всё в одной транзакции, в конце ROLLBACK: тестовые данные не остаются в базе, --- скрипт можно запускать сколько угодно раз. \pset footer off BEGIN; --- Тестовые данные: один большой чат на 15 000 сообщений и 50 маленьких по 100, --- всего 20 000 сообщений. Чужие сообщения в таблице есть, как в настоящей базе. CREATE TEMP TABLE seed_chats AS SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size FROM generate_series(0, 50) AS n; CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; INSERT INTO users (id, username, display_name, password_hash) - SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), 'Explain', 'not-a-real-hash' FROM seed_user; -INSERT INTO chats (id, title) SELECT id, 'explain ' || n FROM seed_chats; + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; INSERT INTO messages (id, chat_id, sender_id, text, created_at) - SELECT gen_random_uuid(), c.id, u.id, 'message ' || g, + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; ANALYZE messages; SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset --- Курсор глубоко в истории: ключ (created_at, id) 10 000-го сообщения большого чата. SELECT created_at AS cursor_time, id AS cursor_id FROM messages WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset diff --git a/student-tests/test_api.py b/student-tests/test_api.py index 382aeeb..0440df6 100644 --- a/student-tests/test_api.py +++ b/student-tests/test_api.py @@ -1,10 +1,3 @@ -"""Собственные тесты API (lab1 + lab2). Запуск одной командой при запущенном стенде: - - python student-tests/test_api.py - -Тесты lab1 переведены с X-User-Id на пароль и Bearer-токен. Добавлены тесты lab2: -вход по Basic, неверный пароль, logout, Bearer на всех защищённых запросах. -""" import base64 import datetime as dt import uuid @@ -16,7 +9,6 @@ def parse_time(value): return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) -# ---------- lab1 (перенесены, теперь с токенами) ---------- def test_duplicate_username(): body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} @@ -79,7 +71,6 @@ def test_pagination(): for i in range(3): send(alice, other["id"], f"Other {i}") first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json - # Курсор от другого чата и испорченный курсор — 400. call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) print("PASS: pagination works across multiple pages, foreign cursor returns 400") @@ -110,7 +101,23 @@ def test_unicode(): print("PASS: unicode text is stored as is") -# ---------- lab2: пароль, вход, Bearer, logout ---------- +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + def test_user_has_no_password(): body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} @@ -155,7 +162,6 @@ def test_wrong_password(): wrong = login(user["username"], "Wrong-password-000", expected=401) unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") - # Одинаковый ответ: по нему нельзя узнать, существует ли такой username. check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") print("PASS: wrong password and unknown user both return the same 401") @@ -196,7 +202,6 @@ def test_x_user_id_ignored(): def test_auth_before_body(): alice = new_user("alice") chat = new_chat(alice) - # Без токена всегда 401, даже если тело неверное или чат чужой: сначала «кто ты». call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) print("PASS: 401 comes before 400 and 404") @@ -225,6 +230,7 @@ if __name__ == "__main__": test_pagination, test_chat_list_pagination, test_unicode, + test_nul_character_is_stored, test_user_has_no_password, test_password_rules, test_login_gives_token, diff --git a/student-tests/test_db.py b/student-tests/test_db.py index 29c4034..3f9c83b 100644 --- a/student-tests/test_db.py +++ b/student-tests/test_db.py @@ -1,14 +1,3 @@ -"""Проверки на уровне базы данных. Запуск при запущенном стенде: - - python student-tests/test_db.py - -SQL выполняется в контейнере postgres через `docker compose exec postgres psql`. -1. Откат составной операции: чат с несуществующим участником -> 400, и в базе - не остаётся ни чата, ни участников, хотя строка чата успела вставиться. -2. База сама не даёт записать сообщение от не-участника (внешний ключ). -3. Пароль хранится как хеш Argon2id, а не как текст или быстрый хеш. -4. Миграции: после перезапуска API ничего не применяется повторно. -""" import time import uuid @@ -26,14 +15,12 @@ def test_create_chat_rolls_back(): title = "rollback-" + uuid.uuid4().hex[:8] inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) - # Второй участник не существует: INSERT чата пройдёт, INSERT участника упадёт на внешнем ключе. call("POST", "/api/v1/chats", 400, token=alice["token"], body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") - # Статистика PostgreSQL считает и отменённые вставки; она обновляется с задержкой до ~1 с. inserted_after = inserted_before deadline = time.monotonic() + 15 while inserted_after == inserted_before and time.monotonic() < deadline: @@ -48,7 +35,7 @@ def test_create_chat_rolls_back(): def test_message_needs_membership_in_db(): code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " - f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', 'direct insert')") + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") print("PASS: the database itself rejects a message from a non-member")