Files
mtusi2026lab3/src/messenger/repo.py
T
Alexey BodanyukandClaude Opus 5.5 4830be414e perf: идемпотентная отправка сообщения одним вызовом SQL (трек 1)
Гипотеза из профиля (py-spy, 2 x 1.0 CPU, 240 req/s): ~57 % CPU реплики
уходит в psycopg — накладные расходы на каждый оператор; на одно
сообщение ~11 обращений к PostgreSQL в двух транзакциях (проверка
членства, BEGIN, ключ, часы чата, сообщение, COMMIT, проверки пула).

- migrations/0003: функция post_message — та же логика и тот же SQL,
  что в многооператорном пути; вызывается в autocommit, поэтому вызов и
  есть транзакция (ключ, часы чата и сообщение — всё или ничего).
- пул: autocommit-соединения; многооператорная работа по-прежнему в явной
  conn.transaction(); ping — один оператор без транзакции.
- MESSAGE_WRITE_PATH=function (по умолчанию) | multi (прежний путь) — A/B
  на одном образе. Порядок ответов одинаков: постороннему всегда 404,
  даже при неверном ключе/теле.
- Оба режима проходят одни и те же тесты идемпотентности и контракта.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 16:03:22 +03:00

329 lines
15 KiB
Python

"""SQL for users, chats and messages. All statements are parameterised (%s placeholders):
values travel separately from the SQL text, so input can never change the query.
Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is
encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md.
"""
from __future__ import annotations
import datetime as dt
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from .cursor import SortKey
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound, Unavailable
from .timefmt import from_us, rfc3339, to_us
def _b(text: str) -> bytes:
return text.encode("utf-8")
def _s(raw: bytes | memoryview) -> str:
return bytes(raw).decode("utf-8")
@dataclass(slots=True, frozen=True)
class User:
id: str
username: str
display_name: str
created_at: dt.datetime
def to_json(self) -> dict:
return {"id": self.id, "username": self.username, "display_name": self.display_name,
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: dt.datetime
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids),
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: dt.datetime
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
"created_at": rfc3339(self.created_at)}
def _user(row) -> User:
return User(str(row[0]), row[1], _s(row[2]), row[3])
# ------------------------------------------------------------------ users
async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User:
async def work(conn: psycopg.AsyncConnection) -> User:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) "
"RETURNING id, username, display_name, created_at",
(uuid.uuid4(), username, _b(display_name), password_hash))
return _user(await cur.fetchone())
try:
return await db.run(work)
except pg_errors.UniqueViolation:
# Two concurrent registrations: the unique index lets exactly one INSERT commit.
raise Conflict("Имя пользователя уже занято") from None
async def get_user(db: Database, user_id: str) -> User | None:
async def work(conn: psycopg.AsyncConnection) -> User | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
row = await cur.fetchone()
return _user(row) if row else None
return await db.run(work)
async def get_credentials(db: Database, username: str) -> tuple[User, str] | None:
async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,))
row = await cur.fetchone()
return (_user(row), row[4]) if row else None
return await db.run(work)
# ------------------------------------------------------------------ chats
async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat:
"""Atomic: the chat row and all memberships commit together or not at all.
The chat is inserted FIRST; an unknown member then fails the membership INSERT
with a foreign-key violation and the whole transaction (chat row included) is
rolled back. student-tests check that nothing is left behind.
"""
if creator_id not in member_ids:
raise InvalidRequest("member_ids должен включать текущего пользователя")
chat_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at",
(chat_id, _b(title), creator_id))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) "
"SELECT %s, m.user_id, m.ord - 1, %s "
"FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
(chat_id, created_at, member_ids))
return created_at
try:
created_at = await db.run(work)
except pg_errors.ForeignKeyViolation:
raise InvalidRequest("member_ids содержит несуществующих пользователей") from None
return Chat(str(chat_id), title, tuple(member_ids), created_at)
async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]:
if not rows:
return []
ids = [r[0] for r in rows]
cur = await conn.execute(
"SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,))
members: dict[uuid.UUID, list[str]] = {i: [] for i in ids}
for chat_id, user_id in await cur.fetchall():
members[chat_id].append(str(user_id))
return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows]
async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat:
"""The chat if the user is a member; a missing chat and a foreign chat are both 404."""
async def work(conn: psycopg.AsyncConnection) -> Chat:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Чат не найден")
return (await _load_chats(conn, [row]))[0]
return await db.run(work)
async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None:
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
if await cur.fetchone() is None:
raise NotFound("Чат не найден")
async def ensure_member(db: Database, user_id: str, chat_id: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await _check_member(conn, user_id, chat_id)
await db.run(work)
async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]:
async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]:
if after is None:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, limit + 1))
else:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) "
"ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, from_us(after[0]), after[1], limit + 1))
rows = await cur.fetchall()
return await _load_chats(conn, rows[:limit]), len(rows) > limit
return await db.run(work)
# ------------------------------------------------------------------ messages
@dataclass(slots=True, frozen=True)
class PostResult:
message: Message
replayed: bool # True: the key was already used with this body, nothing new was written
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
text: str) -> dt.datetime:
"""Insert with a per-chat strictly increasing timestamp.
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
chat are serialised: a message with a smaller created_at can never commit after
a message with a bigger one, and a reader paging by cursor cannot skip it.
"""
cur = await conn.execute(
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
(chat_id,))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
(message_id, chat_id, user_id, _b(text), created_at))
return created_at
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired).
CLAIM_KEY = (
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) "
"VALUES (%s, %s, %s, %s, %s) "
"ON CONFLICT (user_id, chat_id, key) DO UPDATE SET "
"request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() "
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
"RETURNING 1")
STORED_RESULT = (
"SELECT k.request_hash, m.id, m.text, m.created_at FROM idempotency_keys k "
"JOIN messages m ON m.id = k.message_id "
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
ttl_seconds: int) -> PostResult:
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
ever left behind: a crash or 5xx rolls everything back and the client may retry);
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
* same key, other body -> 409; the membership check comes first, so outsiders always get 404.
"""
message_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> PostResult:
await _check_member(conn, user_id, chat_id)
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
if await cur.fetchone() is not None:
created_at = await _insert_message(conn, message_id, user_id, chat_id, text)
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at), False)
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
row = await cur.fetchone()
if row is None:
# The key row vanished between the two statements (expired-key cleanup): retryable.
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
if bytes(row[0]) != request_hash:
raise Conflict("Idempotency-Key уже использован с другим сообщением")
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3]), True)
return await db.run(work)
POST_MESSAGE_CALL = "SELECT o_status, o_id, o_text, o_created_at FROM post_message(%s, %s, %s, %s, %s, %s, %s)"
async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes,
text: str, ttl_seconds: int) -> PostResult:
"""Same contract as post_message, ONE round trip: SQL function post_message (migration 0003)
in autocommit mode, i.e. the call is the whole transaction. Track 1 improvement."""
message_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> tuple:
cur = await conn.execute(POST_MESSAGE_CALL,
(user_id, chat_id, key, request_hash, message_id, _b(text), ttl_seconds))
return await cur.fetchone()
status, stored_id, stored_text, created_at = await db.run(work, transaction=False)
if status == "not_found":
raise NotFound("Чат не найден")
if status == "conflict":
raise Conflict("Idempotency-Key уже использован с другим сообщением")
if status == "retry":
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at), status == "replayed")
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
"""Delete expired keys in bounded batches; one replica at a time (advisory xact lock)."""
async def work(conn: psycopg.AsyncConnection) -> int:
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,))
if not (await cur.fetchone())[0]:
return 0
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY("
"SELECT ctid FROM idempotency_keys "
"WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))",
(ttl_seconds, batch))
return cur.rowcount
return await db.run(work)
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
# would push the cursor into a Filter and late pages would scan the whole prefix.
HISTORY_FIRST = (
"SELECT id, sender_id, text, created_at FROM messages "
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
HISTORY_AFTER = (
"SELECT id, sender_id, text, created_at FROM messages "
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
async def work(conn: psycopg.AsyncConnection) -> list:
if after is None:
cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1))
else:
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
return await cur.fetchall()
rows = await db.run(work)
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3]) for r in rows[:limit]]
return items, len(rows) > limit