diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql new file mode 100644 index 0000000..86196ee --- /dev/null +++ b/migrations/009_message_text_utf8.sql @@ -0,0 +1,6 @@ +-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload. +-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); +ALTER TABLE messages ADD CONSTRAINT messages_text_check + CHECK (octet_length(text) <= 16000); diff --git a/src/store.py b/src/store.py index 298109a..0ddb32d 100644 --- a/src/store.py +++ b/src/store.py @@ -78,7 +78,7 @@ def chat_view(row: dict) -> dict: def message_view(row: dict) -> dict: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), - "sender_id": str(row["sender_id"]), "text": row["text"], + "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"), "created_at": timestamp(row["created_at"]), "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], } @@ -259,7 +259,7 @@ class Store: row = conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " "RETURNING id,chat_id,sender_id,text,created_at", - (uuid4(), chat_uuid, actor_uuid, text), + (uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")), ).fetchone() for position, attachment_id in enumerate(ids): conn.execute( diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py new file mode 100644 index 0000000..4ff79d8 --- /dev/null +++ b/student-tests/test_message_unicode.py @@ -0,0 +1,156 @@ +"""Run with DATABASE_URL pointing to a disposable PostgreSQL database. + +Checks real HTTP writes, persisted history, retries, and migration of old rows. +""" + +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +from uuid import uuid4 + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database") +class MessageUnicodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + cls.addClassCleanup(cls.stop_server) + + @classmethod + def start_server(cls): + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"}, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API exited during startup") + try: + cls.call("GET", "/health/live") + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + cls.stop_server() + raise AssertionError("API did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def call(cls, method, path, body=None, headers=None, expected=200): + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", **(headers or {})}, + ) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as exc: + response = exc + with response: + payload = response.read() + assert response.status == expected, (method, path, response.status, expected, payload) + return json.loads(payload) if payload else None + + def test_http_round_trip_after_restart_and_replay(self): + username, password = "u_" + uuid4().hex[:16], "Test-password-123" + user = self.call("POST", "/api/v1/users", { + "username": username, "display_name": "Unicode", "password": password, + }, expected=201) + session = self.call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(), + }, expected=201) + auth = {"Authorization": "Bearer " + session["token"]} + chat = self.call("POST", "/api/v1/chats", { + "title": "Unicode", "member_ids": [user["id"]], + }, auth, 201) + path = f"/api/v1/chats/{chat['id']}/messages" + texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000] + messages, keys = [], [] + for text in texts: + key = uuid4().hex + message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(message["text"], text) + messages.append(message) + keys.append(key) + for text in ("", "\0" * 4001, "🙂" * 4001): + self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400) + self.stop_server() + self.start_server() + found, cursor = [], None + while True: + page = self.call("GET", path + "?limit=1" + ( + "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else "" + ), headers=auth) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(found, messages) + if LAB >= 3: + for text, message, key in zip(texts, messages, keys): + replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(replay, message) + self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409) + self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages)) + + def test_upgrade_preserves_old_text_and_is_repeatable(self): + schema = "unicode_" + uuid4().hex + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'CREATE SCHEMA "{schema}"') + isolated = make_conninfo(URL, options=f"-c search_path={schema}") + try: + old_text = "Старое 🙂 \\ literal \\u0000" + with psycopg.connect(isolated) as conn: + conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)") + for migration in sorted((ROOT / "migrations").glob("*.sql")): + version = int(migration.name.split("_", 1)[0]) + if version >= 9: + continue + conn.execute(migration.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,)) + user, chat, message = uuid4(), uuid4(), uuid4() + conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,)) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,)) + conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user)) + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text)) + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with psycopg.connect(isolated) as conn: + conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text()) + saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone() + self.assertEqual(saved.decode("utf-8"), old_text) + finally: + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + unittest.main()