fix: preserve NUL characters in persisted messages
Store message text as UTF-8 bytes and migrate existing rows without loss. Cover HTTP round trips, restart persistence, Unicode limits and idempotent retries. Adapt direct database tools to the binary storage format.
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload.
|
||||
-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8');
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check
|
||||
CHECK (octet_length(text) <= 16000);
|
||||
@@ -29,7 +29,7 @@ def main():
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
|
||||
"SELECT gen_random_uuid(), %s, %s, 'synthetic-' || n, "
|
||||
"SELECT gen_random_uuid(), %s, %s, convert_to('synthetic-' || n, 'UTF8'), "
|
||||
"%s::timestamptz + n * interval '1 microsecond' "
|
||||
"FROM generate_series(1,%s) AS n",
|
||||
(chat["id"], user["id"], base, args.count),
|
||||
|
||||
@@ -50,7 +50,7 @@ def main():
|
||||
for name, size, chat_id in chats:
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
|
||||
"SELECT gen_random_uuid(),%s,%s,%s || n,%s::timestamptz + n * interval '1 microsecond' "
|
||||
"SELECT gen_random_uuid(),%s,%s,convert_to(%s || n, 'UTF8'),%s::timestamptz + n * interval '1 microsecond' "
|
||||
"FROM generate_series(1,%s) n",
|
||||
(chat_id, user["id"], name + "-synthetic-", base, size),
|
||||
)
|
||||
@@ -107,7 +107,7 @@ def main():
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
|
||||
"SELECT gen_random_uuid(),%s,%s,'write-synthetic-' || n,"
|
||||
"SELECT gen_random_uuid(),%s,%s,convert_to('write-synthetic-' || n, 'UTF8'),"
|
||||
"clock_timestamp() + n * interval '1 microsecond' "
|
||||
"FROM generate_series(1,500) n",
|
||||
(write_chat, user["id"]),
|
||||
|
||||
+2
-2
@@ -77,7 +77,7 @@ def chat_view(row: dict) -> dict:
|
||||
def message_view(row: dict) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]), "chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]), "text": row["text"],
|
||||
"sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"),
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
}
|
||||
|
||||
@@ -219,7 +219,7 @@ class Store:
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) "
|
||||
"RETURNING id,chat_id,sender_id,text,created_at",
|
||||
(uuid4(), chat_uuid, actor_uuid, text),
|
||||
(uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")),
|
||||
).fetchone()
|
||||
return message_view(row)
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ class DatabaseTests(unittest.TestCase):
|
||||
def test_migration_and_unique_username_race(self):
|
||||
with psycopg.connect(URL) as conn:
|
||||
versions = conn.execute("SELECT version FROM schema_migrations").fetchall()
|
||||
self.assertEqual(versions, [(1,), (2,)])
|
||||
self.assertEqual(versions, [(1,), (2,), (9,)])
|
||||
name = "u_" + uuid4().hex[:16]
|
||||
|
||||
def register(_):
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Run with DATABASE_URL pointing to a disposable PostgreSQL database.
|
||||
|
||||
Checks real HTTP writes, persisted history, retries, and migration of old rows.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LAB = json.loads((ROOT / "lab.json").read_text())["number"]
|
||||
URL = os.environ.get("DATABASE_URL")
|
||||
|
||||
|
||||
@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database")
|
||||
class MessageUnicodeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
cls.port = sock.getsockname()[1]
|
||||
cls.start_server()
|
||||
cls.addClassCleanup(cls.stop_server)
|
||||
|
||||
@classmethod
|
||||
def start_server(cls):
|
||||
cls.server = subprocess.Popen(
|
||||
[sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)],
|
||||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"},
|
||||
)
|
||||
for _ in range(100):
|
||||
if cls.server.poll() is not None:
|
||||
raise AssertionError("API exited during startup")
|
||||
try:
|
||||
cls.call("GET", "/health/live")
|
||||
return
|
||||
except (AssertionError, urllib.error.URLError):
|
||||
time.sleep(0.05)
|
||||
cls.stop_server()
|
||||
raise AssertionError("API did not become ready")
|
||||
|
||||
@classmethod
|
||||
def stop_server(cls):
|
||||
cls.server.terminate()
|
||||
try:
|
||||
cls.server.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls.server.kill()
|
||||
cls.server.wait(timeout=5)
|
||||
|
||||
@classmethod
|
||||
def call(cls, method, path, body=None, headers=None, expected=200):
|
||||
request = urllib.request.Request(
|
||||
f"http://127.0.0.1:{cls.port}{path}", method=method,
|
||||
data=json.dumps(body).encode() if body is not None else None,
|
||||
headers={"Content-Type": "application/json", **(headers or {})},
|
||||
)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=10)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
payload = response.read()
|
||||
assert response.status == expected, (method, path, response.status, expected, payload)
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
def test_http_round_trip_after_restart_and_replay(self):
|
||||
username, password = "u_" + uuid4().hex[:16], "Test-password-123"
|
||||
user = self.call("POST", "/api/v1/users", {
|
||||
"username": username, "display_name": "Unicode", "password": password,
|
||||
}, expected=201)
|
||||
session = self.call("POST", "/api/v1/auth/sessions", headers={
|
||||
"Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(),
|
||||
}, expected=201)
|
||||
auth = {"Authorization": "Bearer " + session["token"]}
|
||||
chat = self.call("POST", "/api/v1/chats", {
|
||||
"title": "Unicode", "member_ids": [user["id"]],
|
||||
}, auth, 201)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000]
|
||||
messages, keys = [], []
|
||||
for text in texts:
|
||||
key = uuid4().hex
|
||||
message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
|
||||
self.assertEqual(message["text"], text)
|
||||
messages.append(message)
|
||||
keys.append(key)
|
||||
for text in ("", "\0" * 4001, "🙂" * 4001):
|
||||
self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400)
|
||||
self.stop_server()
|
||||
self.start_server()
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
page = self.call("GET", path + "?limit=1" + (
|
||||
"&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else ""
|
||||
), headers=auth)
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
self.assertEqual(found, messages)
|
||||
if LAB >= 3:
|
||||
for text, message, key in zip(texts, messages, keys):
|
||||
replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
|
||||
self.assertEqual(replay, message)
|
||||
self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409)
|
||||
self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages))
|
||||
|
||||
def test_upgrade_preserves_old_text_and_is_repeatable(self):
|
||||
schema = "unicode_" + uuid4().hex
|
||||
with psycopg.connect(URL, autocommit=True) as conn:
|
||||
conn.execute(f'CREATE SCHEMA "{schema}"')
|
||||
isolated = make_conninfo(URL, options=f"-c search_path={schema}")
|
||||
try:
|
||||
old_text = "Старое 🙂 \\ literal \\u0000"
|
||||
with psycopg.connect(isolated) as conn:
|
||||
conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)")
|
||||
for migration in sorted((ROOT / "migrations").glob("*.sql")):
|
||||
version = int(migration.name.split("_", 1)[0])
|
||||
if version >= 9:
|
||||
continue
|
||||
conn.execute(migration.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,))
|
||||
user, chat, message = uuid4(), uuid4(), uuid4()
|
||||
conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,))
|
||||
conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,))
|
||||
conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user))
|
||||
conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text))
|
||||
for _ in range(2):
|
||||
subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
|
||||
with psycopg.connect(isolated) as conn:
|
||||
conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text())
|
||||
saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone()
|
||||
self.assertEqual(saved.decode("utf-8"), old_text)
|
||||
finally:
|
||||
with psycopg.connect(URL, autocommit=True) as conn:
|
||||
conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -47,7 +47,7 @@ def samples(conn, chat_id):
|
||||
"ORDER BY created_at,id OFFSET %s LIMIT 1", (UUID(chat_id), offset)
|
||||
).fetchone()
|
||||
assert row is not None, "benchmark chat must contain 80000 messages"
|
||||
found.append({"id": str(row["id"]), "text_sha256": hashlib.sha256(row["text"].encode()).hexdigest()})
|
||||
found.append({"id": str(row["id"]), "text_sha256": hashlib.sha256(row["text"]).hexdigest()})
|
||||
return found
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user