forked from mtusi2026/mtusi2026lab3
Гипотеза из профиля (py-spy, 2 x 1.0 CPU, 240 req/s): ~57 % CPU реплики уходит в psycopg — накладные расходы на каждый оператор; на одно сообщение ~11 обращений к PostgreSQL в двух транзакциях (проверка членства, BEGIN, ключ, часы чата, сообщение, COMMIT, проверки пула). - migrations/0003: функция post_message — та же логика и тот же SQL, что в многооператорном пути; вызывается в autocommit, поэтому вызов и есть транзакция (ключ, часы чата и сообщение — всё или ничего). - пул: autocommit-соединения; многооператорная работа по-прежнему в явной conn.transaction(); ping — один оператор без транзакции. - MESSAGE_WRITE_PATH=function (по умолчанию) | multi (прежний путь) — A/B на одном образе. Порядок ответов одинаков: постороннему всегда 404, даже при неверном ключе/теле. - Оба режима проходят одни и те же тесты идемпотентности и контракта. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
329 lines
15 KiB
Python
329 lines
15 KiB
Python
"""SQL for users, chats and messages. All statements are parameterised (%s placeholders):
|
|
values travel separately from the SQL text, so input can never change the query.
|
|
|
|
Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is
|
|
encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
|
|
import psycopg
|
|
from psycopg import errors as pg_errors
|
|
|
|
from .cursor import SortKey
|
|
from .db import Database
|
|
from .errors import Conflict, InvalidRequest, NotFound, Unavailable
|
|
from .timefmt import from_us, rfc3339, to_us
|
|
|
|
|
|
def _b(text: str) -> bytes:
|
|
return text.encode("utf-8")
|
|
|
|
|
|
def _s(raw: bytes | memoryview) -> str:
|
|
return bytes(raw).decode("utf-8")
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class User:
|
|
id: str
|
|
username: str
|
|
display_name: str
|
|
created_at: dt.datetime
|
|
|
|
def to_json(self) -> dict:
|
|
return {"id": self.id, "username": self.username, "display_name": self.display_name,
|
|
"created_at": rfc3339(self.created_at)}
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class Chat:
|
|
id: str
|
|
title: str
|
|
member_ids: tuple[str, ...]
|
|
created_at: dt.datetime
|
|
|
|
@property
|
|
def key(self) -> SortKey:
|
|
return (to_us(self.created_at), self.id)
|
|
|
|
def to_json(self) -> dict:
|
|
return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids),
|
|
"created_at": rfc3339(self.created_at)}
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class Message:
|
|
id: str
|
|
chat_id: str
|
|
sender_id: str
|
|
text: str
|
|
created_at: dt.datetime
|
|
|
|
@property
|
|
def key(self) -> SortKey:
|
|
return (to_us(self.created_at), self.id)
|
|
|
|
def to_json(self) -> dict:
|
|
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
|
|
"created_at": rfc3339(self.created_at)}
|
|
|
|
|
|
def _user(row) -> User:
|
|
return User(str(row[0]), row[1], _s(row[2]), row[3])
|
|
|
|
|
|
# ------------------------------------------------------------------ users
|
|
async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User:
|
|
async def work(conn: psycopg.AsyncConnection) -> User:
|
|
cur = await conn.execute(
|
|
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) "
|
|
"RETURNING id, username, display_name, created_at",
|
|
(uuid.uuid4(), username, _b(display_name), password_hash))
|
|
return _user(await cur.fetchone())
|
|
try:
|
|
return await db.run(work)
|
|
except pg_errors.UniqueViolation:
|
|
# Two concurrent registrations: the unique index lets exactly one INSERT commit.
|
|
raise Conflict("Имя пользователя уже занято") from None
|
|
|
|
|
|
async def get_user(db: Database, user_id: str) -> User | None:
|
|
async def work(conn: psycopg.AsyncConnection) -> User | None:
|
|
cur = await conn.execute(
|
|
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
|
|
row = await cur.fetchone()
|
|
return _user(row) if row else None
|
|
return await db.run(work)
|
|
|
|
|
|
async def get_credentials(db: Database, username: str) -> tuple[User, str] | None:
|
|
async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None:
|
|
cur = await conn.execute(
|
|
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
|
(username,))
|
|
row = await cur.fetchone()
|
|
return (_user(row), row[4]) if row else None
|
|
return await db.run(work)
|
|
|
|
|
|
# ------------------------------------------------------------------ chats
|
|
async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat:
|
|
"""Atomic: the chat row and all memberships commit together or not at all.
|
|
|
|
The chat is inserted FIRST; an unknown member then fails the membership INSERT
|
|
with a foreign-key violation and the whole transaction (chat row included) is
|
|
rolled back. student-tests check that nothing is left behind.
|
|
"""
|
|
if creator_id not in member_ids:
|
|
raise InvalidRequest("member_ids должен включать текущего пользователя")
|
|
chat_id = uuid.uuid4()
|
|
|
|
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
|
|
cur = await conn.execute(
|
|
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at",
|
|
(chat_id, _b(title), creator_id))
|
|
(created_at,) = await cur.fetchone()
|
|
await conn.execute(
|
|
"INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) "
|
|
"SELECT %s, m.user_id, m.ord - 1, %s "
|
|
"FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
|
|
(chat_id, created_at, member_ids))
|
|
return created_at
|
|
try:
|
|
created_at = await db.run(work)
|
|
except pg_errors.ForeignKeyViolation:
|
|
raise InvalidRequest("member_ids содержит несуществующих пользователей") from None
|
|
return Chat(str(chat_id), title, tuple(member_ids), created_at)
|
|
|
|
|
|
async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]:
|
|
if not rows:
|
|
return []
|
|
ids = [r[0] for r in rows]
|
|
cur = await conn.execute(
|
|
"SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,))
|
|
members: dict[uuid.UUID, list[str]] = {i: [] for i in ids}
|
|
for chat_id, user_id in await cur.fetchall():
|
|
members[chat_id].append(str(user_id))
|
|
return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows]
|
|
|
|
|
|
async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat:
|
|
"""The chat if the user is a member; a missing chat and a foreign chat are both 404."""
|
|
async def work(conn: psycopg.AsyncConnection) -> Chat:
|
|
cur = await conn.execute(
|
|
"SELECT c.id, c.title, c.created_at FROM chats c "
|
|
"JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
|
|
(user_id, chat_id))
|
|
row = await cur.fetchone()
|
|
if row is None:
|
|
raise NotFound("Чат не найден")
|
|
return (await _load_chats(conn, [row]))[0]
|
|
return await db.run(work)
|
|
|
|
|
|
async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None:
|
|
cur = await conn.execute(
|
|
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
|
|
if await cur.fetchone() is None:
|
|
raise NotFound("Чат не найден")
|
|
|
|
|
|
async def ensure_member(db: Database, user_id: str, chat_id: str) -> None:
|
|
async def work(conn: psycopg.AsyncConnection) -> None:
|
|
await _check_member(conn, user_id, chat_id)
|
|
await db.run(work)
|
|
|
|
|
|
async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]:
|
|
async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]:
|
|
if after is None:
|
|
cur = await conn.execute(
|
|
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
|
|
"WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
|
|
(user_id, limit + 1))
|
|
else:
|
|
cur = await conn.execute(
|
|
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
|
|
"WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) "
|
|
"ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
|
|
(user_id, from_us(after[0]), after[1], limit + 1))
|
|
rows = await cur.fetchall()
|
|
return await _load_chats(conn, rows[:limit]), len(rows) > limit
|
|
return await db.run(work)
|
|
|
|
|
|
# ------------------------------------------------------------------ messages
|
|
@dataclass(slots=True, frozen=True)
|
|
class PostResult:
|
|
message: Message
|
|
replayed: bool # True: the key was already used with this body, nothing new was written
|
|
|
|
|
|
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
|
|
text: str) -> dt.datetime:
|
|
"""Insert with a per-chat strictly increasing timestamp.
|
|
|
|
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
|
|
chat are serialised: a message with a smaller created_at can never commit after
|
|
a message with a bigger one, and a reader paging by cursor cannot skip it.
|
|
"""
|
|
cur = await conn.execute(
|
|
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
|
|
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
|
|
(chat_id,))
|
|
(created_at,) = await cur.fetchone()
|
|
await conn.execute(
|
|
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
|
|
(message_id, chat_id, user_id, _b(text), created_at))
|
|
return created_at
|
|
|
|
|
|
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
|
|
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
|
|
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
|
|
# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired).
|
|
CLAIM_KEY = (
|
|
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) "
|
|
"VALUES (%s, %s, %s, %s, %s) "
|
|
"ON CONFLICT (user_id, chat_id, key) DO UPDATE SET "
|
|
"request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() "
|
|
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
|
|
"RETURNING 1")
|
|
STORED_RESULT = (
|
|
"SELECT k.request_hash, m.id, m.text, m.created_at FROM idempotency_keys k "
|
|
"JOIN messages m ON m.id = k.message_id "
|
|
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
|
|
|
|
|
|
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
|
|
ttl_seconds: int) -> PostResult:
|
|
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
|
|
|
|
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
|
|
ever left behind: a crash or 5xx rolls everything back and the client may retry);
|
|
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
|
|
* same key, other body -> 409; the membership check comes first, so outsiders always get 404.
|
|
"""
|
|
message_id = uuid.uuid4()
|
|
|
|
async def work(conn: psycopg.AsyncConnection) -> PostResult:
|
|
await _check_member(conn, user_id, chat_id)
|
|
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
|
|
if await cur.fetchone() is not None:
|
|
created_at = await _insert_message(conn, message_id, user_id, chat_id, text)
|
|
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at), False)
|
|
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
|
|
row = await cur.fetchone()
|
|
if row is None:
|
|
# The key row vanished between the two statements (expired-key cleanup): retryable.
|
|
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
|
|
if bytes(row[0]) != request_hash:
|
|
raise Conflict("Idempotency-Key уже использован с другим сообщением")
|
|
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3]), True)
|
|
return await db.run(work)
|
|
|
|
|
|
POST_MESSAGE_CALL = "SELECT o_status, o_id, o_text, o_created_at FROM post_message(%s, %s, %s, %s, %s, %s, %s)"
|
|
|
|
|
|
async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes,
|
|
text: str, ttl_seconds: int) -> PostResult:
|
|
"""Same contract as post_message, ONE round trip: SQL function post_message (migration 0003)
|
|
in autocommit mode, i.e. the call is the whole transaction. Track 1 improvement."""
|
|
message_id = uuid.uuid4()
|
|
|
|
async def work(conn: psycopg.AsyncConnection) -> tuple:
|
|
cur = await conn.execute(POST_MESSAGE_CALL,
|
|
(user_id, chat_id, key, request_hash, message_id, _b(text), ttl_seconds))
|
|
return await cur.fetchone()
|
|
status, stored_id, stored_text, created_at = await db.run(work, transaction=False)
|
|
if status == "not_found":
|
|
raise NotFound("Чат не найден")
|
|
if status == "conflict":
|
|
raise Conflict("Idempotency-Key уже использован с другим сообщением")
|
|
if status == "retry":
|
|
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
|
|
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at), status == "replayed")
|
|
|
|
|
|
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
|
|
"""Delete expired keys in bounded batches; one replica at a time (advisory xact lock)."""
|
|
async def work(conn: psycopg.AsyncConnection) -> int:
|
|
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,))
|
|
if not (await cur.fetchone())[0]:
|
|
return 0
|
|
cur = await conn.execute(
|
|
"DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY("
|
|
"SELECT ctid FROM idempotency_keys "
|
|
"WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))",
|
|
(ttl_seconds, batch))
|
|
return cur.rowcount
|
|
return await db.run(work)
|
|
|
|
|
|
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
|
|
# would push the cursor into a Filter and late pages would scan the whole prefix.
|
|
HISTORY_FIRST = (
|
|
"SELECT id, sender_id, text, created_at FROM messages "
|
|
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
|
|
HISTORY_AFTER = (
|
|
"SELECT id, sender_id, text, created_at FROM messages "
|
|
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
|
|
|
|
|
|
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
|
|
async def work(conn: psycopg.AsyncConnection) -> list:
|
|
if after is None:
|
|
cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1))
|
|
else:
|
|
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
|
|
return await cur.fetchall()
|
|
rows = await db.run(work)
|
|
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3]) for r in rows[:limit]]
|
|
return items, len(rows) > limit
|