Store message text as UTF-8 bytes and migrate existing rows without loss. Cover HTTP round trips, restart persistence, Unicode limits and idempotent retries. Adapt direct database tools to the binary storage format. (cherry picked from commit 386276532730b8758298080b49db63738ca300a4)
157 lines
6.8 KiB
Python
157 lines
6.8 KiB
Python
"""Run with DATABASE_URL pointing to a disposable PostgreSQL database.
|
|
|
|
Checks real HTTP writes, persisted history, retries, and migration of old rows.
|
|
"""
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import unittest
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
from psycopg.conninfo import make_conninfo
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
LAB = json.loads((ROOT / "lab.json").read_text())["number"]
|
|
URL = os.environ.get("DATABASE_URL")
|
|
|
|
|
|
@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database")
|
|
class MessageUnicodeTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
cls.port = sock.getsockname()[1]
|
|
cls.start_server()
|
|
cls.addClassCleanup(cls.stop_server)
|
|
|
|
@classmethod
|
|
def start_server(cls):
|
|
cls.server = subprocess.Popen(
|
|
[sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)],
|
|
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"},
|
|
)
|
|
for _ in range(100):
|
|
if cls.server.poll() is not None:
|
|
raise AssertionError("API exited during startup")
|
|
try:
|
|
cls.call("GET", "/health/live")
|
|
return
|
|
except (AssertionError, urllib.error.URLError):
|
|
time.sleep(0.05)
|
|
cls.stop_server()
|
|
raise AssertionError("API did not become ready")
|
|
|
|
@classmethod
|
|
def stop_server(cls):
|
|
cls.server.terminate()
|
|
try:
|
|
cls.server.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
cls.server.kill()
|
|
cls.server.wait(timeout=5)
|
|
|
|
@classmethod
|
|
def call(cls, method, path, body=None, headers=None, expected=200):
|
|
request = urllib.request.Request(
|
|
f"http://127.0.0.1:{cls.port}{path}", method=method,
|
|
data=json.dumps(body).encode() if body is not None else None,
|
|
headers={"Content-Type": "application/json", **(headers or {})},
|
|
)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=10)
|
|
except urllib.error.HTTPError as exc:
|
|
response = exc
|
|
with response:
|
|
payload = response.read()
|
|
assert response.status == expected, (method, path, response.status, expected, payload)
|
|
return json.loads(payload) if payload else None
|
|
|
|
def test_http_round_trip_after_restart_and_replay(self):
|
|
username, password = "u_" + uuid4().hex[:16], "Test-password-123"
|
|
user = self.call("POST", "/api/v1/users", {
|
|
"username": username, "display_name": "Unicode", "password": password,
|
|
}, expected=201)
|
|
session = self.call("POST", "/api/v1/auth/sessions", headers={
|
|
"Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(),
|
|
}, expected=201)
|
|
auth = {"Authorization": "Bearer " + session["token"]}
|
|
chat = self.call("POST", "/api/v1/chats", {
|
|
"title": "Unicode", "member_ids": [user["id"]],
|
|
}, auth, 201)
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000]
|
|
messages, keys = [], []
|
|
for text in texts:
|
|
key = uuid4().hex
|
|
message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
|
|
self.assertEqual(message["text"], text)
|
|
messages.append(message)
|
|
keys.append(key)
|
|
for text in ("", "\0" * 4001, "🙂" * 4001):
|
|
self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400)
|
|
self.stop_server()
|
|
self.start_server()
|
|
found, cursor = [], None
|
|
while True:
|
|
page = self.call("GET", path + "?limit=1" + (
|
|
"&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else ""
|
|
), headers=auth)
|
|
found.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
break
|
|
self.assertEqual(found, messages)
|
|
if LAB >= 3:
|
|
for text, message, key in zip(texts, messages, keys):
|
|
replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
|
|
self.assertEqual(replay, message)
|
|
self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409)
|
|
self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages))
|
|
|
|
def test_upgrade_preserves_old_text_and_is_repeatable(self):
|
|
schema = "unicode_" + uuid4().hex
|
|
with psycopg.connect(URL, autocommit=True) as conn:
|
|
conn.execute(f'CREATE SCHEMA "{schema}"')
|
|
isolated = make_conninfo(URL, options=f"-c search_path={schema}")
|
|
try:
|
|
old_text = "Старое 🙂 \\ literal \\u0000"
|
|
with psycopg.connect(isolated) as conn:
|
|
conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)")
|
|
for migration in sorted((ROOT / "migrations").glob("*.sql")):
|
|
version = int(migration.name.split("_", 1)[0])
|
|
if version >= 9:
|
|
continue
|
|
conn.execute(migration.read_text())
|
|
conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,))
|
|
user, chat, message = uuid4(), uuid4(), uuid4()
|
|
conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,))
|
|
conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,))
|
|
conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user))
|
|
conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text))
|
|
for _ in range(2):
|
|
subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
|
|
with psycopg.connect(isolated) as conn:
|
|
conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text())
|
|
saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone()
|
|
self.assertEqual(saved.decode("utf-8"), old_text)
|
|
finally:
|
|
with psycopg.connect(URL, autocommit=True) as conn:
|
|
conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|