From 386276532730b8758298080b49db63738ca300a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sat, 7 Nov 2026 09:51:39 +0300 Subject: [PATCH] fix: preserve NUL characters in persisted messages Store message text as UTF-8 bytes and migrate existing rows without loss. Cover HTTP round trips, restart persistence, Unicode limits and idempotent retries. Adapt direct database tools to the binary storage format. --- migrations/009_message_text_utf8.sql | 6 + scripts/seed_history.py | 2 +- scripts/track2_experiment.py | 4 +- src/store.py | 4 +- student-tests/test_database.py | 2 +- student-tests/test_message_unicode.py | 156 ++++++++++++++++++++++++++ student-tests/test_restore.py | 2 +- 7 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 migrations/009_message_text_utf8.sql create mode 100644 student-tests/test_message_unicode.py diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql new file mode 100644 index 0000000..86196ee --- /dev/null +++ b/migrations/009_message_text_utf8.sql @@ -0,0 +1,6 @@ +-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload. +-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); +ALTER TABLE messages ADD CONSTRAINT messages_text_check + CHECK (octet_length(text) <= 16000); diff --git a/scripts/seed_history.py b/scripts/seed_history.py index d257035..5d56404 100644 --- a/scripts/seed_history.py +++ b/scripts/seed_history.py @@ -29,7 +29,7 @@ def main(): with connection() as conn: conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text,created_at) " - "SELECT gen_random_uuid(), %s, %s, 'synthetic-' || n, " + "SELECT gen_random_uuid(), %s, %s, convert_to('synthetic-' || n, 'UTF8'), " "%s::timestamptz + n * interval '1 microsecond' " "FROM generate_series(1,%s) AS n", (chat["id"], user["id"], base, args.count), diff --git a/scripts/track2_experiment.py b/scripts/track2_experiment.py index 8804dac..2c48985 100644 --- a/scripts/track2_experiment.py +++ b/scripts/track2_experiment.py @@ -50,7 +50,7 @@ def main(): for name, size, chat_id in chats: conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text,created_at) " - "SELECT gen_random_uuid(),%s,%s,%s || n,%s::timestamptz + n * interval '1 microsecond' " + "SELECT gen_random_uuid(),%s,%s,convert_to(%s || n, 'UTF8'),%s::timestamptz + n * interval '1 microsecond' " "FROM generate_series(1,%s) n", (chat_id, user["id"], name + "-synthetic-", base, size), ) @@ -107,7 +107,7 @@ def main(): with conn.transaction(): conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text,created_at) " - "SELECT gen_random_uuid(),%s,%s,'write-synthetic-' || n," + "SELECT gen_random_uuid(),%s,%s,convert_to('write-synthetic-' || n, 'UTF8')," "clock_timestamp() + n * interval '1 microsecond' " "FROM generate_series(1,500) n", (write_chat, user["id"]), diff --git a/src/store.py b/src/store.py index 6f43bda..1df017f 100644 --- a/src/store.py +++ b/src/store.py @@ -77,7 +77,7 @@ def chat_view(row: dict) -> dict: def message_view(row: dict) -> dict: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), - "sender_id": str(row["sender_id"]), "text": row["text"], + "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"), "created_at": timestamp(row["created_at"]), } @@ -219,7 +219,7 @@ class Store: row = conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " "RETURNING id,chat_id,sender_id,text,created_at", - (uuid4(), chat_uuid, actor_uuid, text), + (uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")), ).fetchone() return message_view(row) diff --git a/student-tests/test_database.py b/student-tests/test_database.py index 49e3d6a..908da76 100644 --- a/student-tests/test_database.py +++ b/student-tests/test_database.py @@ -33,7 +33,7 @@ class DatabaseTests(unittest.TestCase): def test_migration_and_unique_username_race(self): with psycopg.connect(URL) as conn: versions = conn.execute("SELECT version FROM schema_migrations").fetchall() - self.assertEqual(versions, [(1,), (2,)]) + self.assertEqual(versions, [(1,), (2,), (9,)]) name = "u_" + uuid4().hex[:16] def register(_): diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py new file mode 100644 index 0000000..4ff79d8 --- /dev/null +++ b/student-tests/test_message_unicode.py @@ -0,0 +1,156 @@ +"""Run with DATABASE_URL pointing to a disposable PostgreSQL database. + +Checks real HTTP writes, persisted history, retries, and migration of old rows. +""" + +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +from uuid import uuid4 + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database") +class MessageUnicodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + cls.addClassCleanup(cls.stop_server) + + @classmethod + def start_server(cls): + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"}, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API exited during startup") + try: + cls.call("GET", "/health/live") + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + cls.stop_server() + raise AssertionError("API did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def call(cls, method, path, body=None, headers=None, expected=200): + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", **(headers or {})}, + ) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as exc: + response = exc + with response: + payload = response.read() + assert response.status == expected, (method, path, response.status, expected, payload) + return json.loads(payload) if payload else None + + def test_http_round_trip_after_restart_and_replay(self): + username, password = "u_" + uuid4().hex[:16], "Test-password-123" + user = self.call("POST", "/api/v1/users", { + "username": username, "display_name": "Unicode", "password": password, + }, expected=201) + session = self.call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(), + }, expected=201) + auth = {"Authorization": "Bearer " + session["token"]} + chat = self.call("POST", "/api/v1/chats", { + "title": "Unicode", "member_ids": [user["id"]], + }, auth, 201) + path = f"/api/v1/chats/{chat['id']}/messages" + texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000] + messages, keys = [], [] + for text in texts: + key = uuid4().hex + message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(message["text"], text) + messages.append(message) + keys.append(key) + for text in ("", "\0" * 4001, "🙂" * 4001): + self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400) + self.stop_server() + self.start_server() + found, cursor = [], None + while True: + page = self.call("GET", path + "?limit=1" + ( + "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else "" + ), headers=auth) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(found, messages) + if LAB >= 3: + for text, message, key in zip(texts, messages, keys): + replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(replay, message) + self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409) + self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages)) + + def test_upgrade_preserves_old_text_and_is_repeatable(self): + schema = "unicode_" + uuid4().hex + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'CREATE SCHEMA "{schema}"') + isolated = make_conninfo(URL, options=f"-c search_path={schema}") + try: + old_text = "Старое 🙂 \\ literal \\u0000" + with psycopg.connect(isolated) as conn: + conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)") + for migration in sorted((ROOT / "migrations").glob("*.sql")): + version = int(migration.name.split("_", 1)[0]) + if version >= 9: + continue + conn.execute(migration.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,)) + user, chat, message = uuid4(), uuid4(), uuid4() + conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,)) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,)) + conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user)) + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text)) + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with psycopg.connect(isolated) as conn: + conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text()) + saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone() + self.assertEqual(saved.decode("utf-8"), old_text) + finally: + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_restore.py b/student-tests/test_restore.py index 4312aae..9214516 100644 --- a/student-tests/test_restore.py +++ b/student-tests/test_restore.py @@ -47,7 +47,7 @@ def samples(conn, chat_id): "ORDER BY created_at,id OFFSET %s LIMIT 1", (UUID(chat_id), offset) ).fetchone() assert row is not None, "benchmark chat must contain 80000 messages" - found.append({"id": str(row["id"]), "text_sha256": hashlib.sha256(row["text"].encode()).hexdigest()}) + found.append({"id": str(row["id"]), "text_sha256": hashlib.sha256(row["text"]).hexdigest()}) return found