"""Доступ к PostgreSQL. Все запросы параметризованы (%s), SQL не склеивается со значениями пользователя. Каждая публичная операция — одна транзакция (`with pool.connection()` делает COMMIT при успехе и ROLLBACK при исключении). """ from datetime import datetime from typing import List, Optional, Sequence, Tuple from uuid import UUID, uuid4 from psycopg import errors from psycopg.rows import dict_row from psycopg.types.json import Jsonb from psycopg_pool import ConnectionPool from .models import Attachment, Chat, Message, User, Variant class UsernameTaken(Exception): pass class UnknownMember(Exception): pass class NotChatMember(Exception): """Чата нет или пользователь не участник -> 404.""" class IdempotencyConflict(Exception): """Тот же ключ уже использован с другим телом -> 409.""" class IdempotencyRace(Exception): """Строка ключа исчезла между шагами -> 503, клиент повторяет с тем же ключом.""" class AttachmentNotFound(Exception): """Вложение не существует, ещё не принято (pending) или принадлежит другому чату -> 404.""" class AttachmentFailed(Exception): """lab5: вложение в состоянии failed нельзя прикрепить к сообщению -> 409.""" # Одна реплика за раз выполняет сборку мусора вложений (src/reconcile.py). ATTACHMENT_GC_LOCK_ID = 2026_0004 ATTACHMENT_COLUMNS = ( "a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " "a.sha256, a.state, a.error_code, a.pipeline_version, a.created_at, a.updated_at" ) # Внутреннее состояние -> публичный status. pending (приём ещё не завершён) наружу # не попадает: такие строки запросы API не выбирают. PUBLIC_STATES = ("queued", "processing", "ready", "failed") # Подзапрос: attachment_ids сообщения в исходном порядке (position). MESSAGE_ATTACHMENT_IDS = ( "COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) " "FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[]) AS attachment_ids" ) def _attachment(row, variants=()) -> Attachment: state = row["state"] return Attachment( id=row["id"], chat_id=row["chat_id"], owner_id=row["owner_id"], filename=row["filename"], content_type=row["content_type"], size_bytes=row["size_bytes"], sha256=row["sha256"], status=state, # Варианты публикуются вместе со статусом ready (одна транзакция worker). variants=list(variants) if state == "ready" else [], error_code=row.get("error_code") if state == "failed" else None, created_at=row["created_at"], updated_at=row["updated_at"], ) # Одна реплика за раз чистит истёкшие ключи идемпотентности. IDEMPOTENCY_CLEANUP_LOCK_ID = 2026_0003 Position = Optional[Tuple[datetime, UUID]] def _user(row) -> User: return User( id=row["id"], username=row["username"], display_name=row["display_name"], created_at=row["created_at"], ) class Storage: def __init__(self, pool: ConnectionPool) -> None: self.pool = pool def ping(self) -> None: with self.pool.connection() as conn: conn.execute("SELECT 1") # users def create_user(self, username: str, display_name: str, password_hash: str) -> User: try: with self.pool.connection() as conn: row = conn.cursor(row_factory=dict_row).execute( """ INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) RETURNING id, username, display_name, created_at """, (uuid4(), username, display_name, password_hash), ).fetchone() except errors.UniqueViolation: # Гонку двух одинаковых регистраций решает UNIQUE(username) в БД, # а не проверка «SELECT, потом INSERT» в приложении. raise UsernameTaken() return _user(row) def get_credentials(self, username: str) -> Optional[Tuple[User, str]]: with self.pool.connection() as conn: row = conn.cursor(row_factory=dict_row).execute( """ SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s """, (username,), ).fetchone() if row is None: return None return _user(row), row["password_hash"] # sessions (источник истины) def create_session(self, token_hash: bytes, user_id: UUID, ttl_seconds: int) -> datetime: with self.pool.connection() as conn: # Заодно убираем давно истёкшие сессии этого пользователя. conn.execute( "DELETE FROM sessions WHERE user_id = %s AND expires_at < now() - interval '1 day'", (user_id,), ) row = conn.execute( """ INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at """, (token_hash, user_id, ttl_seconds), ).fetchone() return row[0] def find_session(self, token_hash: bytes) -> Optional[Tuple[User, datetime]]: with self.pool.connection() as conn: row = conn.cursor(row_factory=dict_row).execute( """ SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp() """, (token_hash,), ).fetchone() if row is None: return None return _user(row), row["expires_at"] def revoke_session(self, token_hash: bytes) -> bool: """Отзыв и долг «инвалидировать кэш» фиксируются одним COMMIT.""" with self.pool.connection() as conn: with conn.transaction(): row = conn.execute( """ UPDATE sessions SET revoked_at = clock_timestamp() WHERE token_hash = %s AND revoked_at IS NULL AND expires_at > clock_timestamp() RETURNING 1 """, (token_hash,), ).fetchone() if row is not None: conn.execute( """ INSERT INTO session_cache_invalidations (token_hash) VALUES (%s) ON CONFLICT (token_hash) DO NOTHING """, (token_hash,), ) return row is not None # долг инвалидации кэша def has_pending_invalidations(self) -> bool: with self.pool.connection() as conn: return conn.execute( "SELECT EXISTS (SELECT 1 FROM session_cache_invalidations)" ).fetchone()[0] def pending_invalidations(self, limit: int = 500) -> List[Tuple[bytes, float]]: """[(token_hash, возраст в секундах)] — самые старые первыми.""" with self.pool.connection() as conn: rows = conn.execute( """ SELECT token_hash, EXTRACT(EPOCH FROM clock_timestamp() - revoked_at) FROM session_cache_invalidations ORDER BY revoked_at LIMIT %s """, (limit,), ).fetchall() return [(bytes(h), float(age)) for h, age in rows] def delete_invalidations(self, token_hashes: Sequence[bytes]) -> None: if not token_hashes: return with self.pool.connection() as conn: conn.execute( "DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)", (list(token_hashes),), ) # chats def _insert_members(self, conn, chat_id: UUID, member_ids: Sequence[UUID]) -> None: with conn.cursor() as cur: cur.executemany( "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", [(chat_id, member_id, pos) for pos, member_id in enumerate(member_ids)], ) def create_chat(self, title: str, member_ids: Sequence[UUID]) -> Chat: """Чат и все участники создаются в одной транзакции. Существование участников проверяет внешний ключ chat_members.user_id: если хоть один id неизвестен, INSERT падает, и откатывается вся транзакция, включая уже вставленную строку chats и других участников. """ chat_id = uuid4() try: with self.pool.connection() as conn: with conn.transaction(): created_at = conn.execute( "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING created_at", (chat_id, title), ).fetchone()[0] self._insert_members(conn, chat_id, member_ids) except errors.ForeignKeyViolation: raise UnknownMember() return Chat(id=chat_id, title=title, member_ids=list(member_ids), created_at=created_at) def _members_for(self, conn, chat_ids: List[UUID]) -> dict: members = {chat_id: [] for chat_id in chat_ids} if not chat_ids: return members rows = conn.execute( """ SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position """, (chat_ids,), ).fetchall() for chat_id, user_id in rows: members[chat_id].append(user_id) return members def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> Optional[Chat]: with self.pool.connection() as conn: row = conn.cursor(row_factory=dict_row).execute( """ SELECT c.id, c.title, c.created_at FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s """, (user_id, chat_id), ).fetchone() if row is None: return None members = self._members_for(conn, [row["id"]]) return Chat(id=row["id"], title=row["title"], member_ids=members[row["id"]], created_at=row["created_at"]) def list_user_chats(self, user_id: UUID, limit: int, after: Position) -> Tuple[List[Chat], bool]: params: list = [user_id] keyset = "" if after is not None: keyset = "AND (c.created_at, c.id) > (%s, %s)" params.extend(after) params.append(limit + 1) with self.pool.connection() as conn: rows = conn.cursor(row_factory=dict_row).execute( """ SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = %s {keyset} ORDER BY c.created_at, c.id LIMIT %s """.format(keyset=keyset), params, ).fetchall() has_more = len(rows) > limit rows = rows[:limit] members = self._members_for(conn, [r["id"] for r in rows]) chats = [ Chat(id=r["id"], title=r["title"], member_ids=members[r["id"]], created_at=r["created_at"]) for r in rows ] return chats, has_more # messages @staticmethod def _insert_message(conn, message_id: UUID, chat_id: UUID, sender_id: UUID, text: str) -> Optional[Message]: """Проверка членства и вставка — один оператор, без окна гонки.""" row = conn.cursor(row_factory=dict_row).execute( """ INSERT INTO messages (id, chat_id, sender_id, text) SELECT %s, %s, %s, %s WHERE EXISTS ( SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s ) RETURNING id, chat_id, sender_id, text, created_at """, (message_id, chat_id, sender_id, text, chat_id, sender_id), ).fetchone() return Message(**row) if row is not None else None @staticmethod def _link_attachments(conn, message: Message, attachment_ids: Sequence[UUID]) -> None: """Привязка вложений в той же транзакции, что и сообщение. lab5: можно ссылаться на queued/processing/ready вложения ЭТОГО чата. Неизвестное/чужое/ещё не принятое (pending) -> AttachmentNotFound (404); failed -> AttachmentFailed (409). Любое исключение -> ROLLBACK сообщения и ключа идемпотентности. FOR KEY SHARE не даёт удалить строку до COMMIT; составной FK (attachment_id, chat_id) дублирует проверку чата в БД. """ if not attachment_ids: return ids = list(attachment_ids) rows = conn.execute( """ SELECT id, state FROM attachments WHERE id = ANY(%s) AND chat_id = %s AND state <> 'pending' FOR KEY SHARE """, (ids, message.chat_id), ).fetchall() if len(rows) != len(ids): raise AttachmentNotFound() if any(state == "failed" for _id, state in rows): raise AttachmentFailed() with conn.cursor() as cur: cur.executemany( """ INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) VALUES (%s, %s, %s, %s) """, [(message.id, aid, message.chat_id, pos) for pos, aid in enumerate(ids)], ) message.attachment_ids = ids def create_message( self, chat_id: UUID, sender_id: UUID, text: str, idem_key: str, request_hash: bytes, idem_ttl_seconds: int, attachment_ids: Sequence[UUID] = (), ) -> Tuple[Message, bool]: """Идемпотентная отправка. Возвращает (сообщение, replayed). Всё — одна транзакция READ COMMITTED (см. migrations/003): 1. INSERT строки ключа. Если такой ключ уже есть и он не истёк, ON CONFLICT ... WHERE ничего не меняет и RETURNING пуст. Если конкурентная транзакция с тем же ключом ещё не завершилась, INSERT ЖДЁТ её COMMIT/ROLLBACK на уникальном индексе — поэтому одновременные повторы через разные реплики создают ровно одно сообщение. 2a. Ключ наш -> INSERT сообщения (с проверкой членства). Не участник -> исключение -> ROLLBACK: ключ не «сгорает» на 404. 2b. Ключ чужой (уже подтверждён) -> снова проверяем доступ, сравниваем hash тела: совпал -> то же сообщение; нет -> 409. Ошибка/обрыв на любом шаге откатывает и ключ, и сообщение. """ new_id = uuid4() with self.pool.connection() as conn: with conn.transaction(): owned = conn.execute( """ INSERT INTO idempotency_keys (user_id, method, chat_id, idem_key, request_hash, message_id, expires_at) VALUES (%s, 'POST', %s, %s, %s, %s, clock_timestamp() + make_interval(secs => %s)) ON CONFLICT (user_id, method, chat_id, idem_key) DO UPDATE SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp(), expires_at = EXCLUDED.expires_at -- истёкший (ещё не удалённый очисткой) ключ можно занять заново WHERE idempotency_keys.expires_at <= clock_timestamp() RETURNING message_id """, (sender_id, chat_id, idem_key, request_hash, new_id, idem_ttl_seconds), ).fetchone() if owned is not None: message = self._insert_message(conn, new_id, chat_id, sender_id, text) if message is None: raise NotChatMember() # -> ROLLBACK вместе со строкой ключа # lab4: чужое/неготовое вложение -> исключение -> ROLLBACK всего. self._link_attachments(conn, message, attachment_ids) return message, False # Ключ уже подтверждён другой (возможно, конкурентной) транзакцией. member = conn.execute( "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, sender_id), ).fetchone() if member is None: raise NotChatMember() row = conn.cursor(row_factory=dict_row).execute( """ SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, {attachments} FROM idempotency_keys k JOIN messages m ON m.id = k.message_id WHERE k.user_id = %s AND k.method = 'POST' AND k.chat_id = %s AND k.idem_key = %s """.format(attachments=MESSAGE_ATTACHMENT_IDS), (sender_id, chat_id, idem_key), ).fetchone() if row is None: # Теоретически: строка удалена между шагами (очистка/каскад). # Отдаём 503-подобную транзиентную ошибку, клиент повторит с тем же ключом. raise IdempotencyRace() if bytes(row.pop("request_hash")) != request_hash: raise IdempotencyConflict() return Message(**row), True def delete_expired_idempotency_keys(self, batch: int = 1000) -> Optional[int]: """Удаляет истёкшие ключи пачкой. None — очистку сейчас выполняет другая реплика.""" with self.pool.connection() as conn: with conn.transaction(): got = conn.execute( "SELECT pg_try_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,) ).fetchone()[0] if not got: return None cur = conn.execute( """ DELETE FROM idempotency_keys WHERE ctid IN ( SELECT ctid FROM idempotency_keys WHERE expires_at <= clock_timestamp() LIMIT %s ) """, (batch,), ) return cur.rowcount def list_messages( self, chat_id: UUID, user_id: UUID, limit: int, after: Position ) -> Optional[Tuple[List[Message], bool]]: params: list = [chat_id] keyset = "" if after is not None: keyset = "AND (created_at, id) > (%s, %s)" params.extend(after) params.append(limit + 1) with self.pool.connection() as conn: member = conn.execute( "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id), ).fetchone() if member is None: return None # Keyset-пагинация по индексу messages_chat_created_id_idx: # Index Scan сразу с позиции курсора, читается limit+1 строк. rows = conn.cursor(row_factory=dict_row).execute( """ SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, {attachments} FROM messages m WHERE m.chat_id = %s {keyset} ORDER BY m.created_at, m.id LIMIT %s """.format(keyset=keyset.replace("(created_at, id)", "(m.created_at, m.id)"), attachments=MESSAGE_ATTACHMENT_IDS), params, ).fetchall() has_more = len(rows) > limit return [Message(**r) for r in rows[:limit]], has_more # attachments (lab4) def is_chat_member(self, chat_id: UUID, user_id: UUID) -> bool: with self.pool.connection() as conn: return conn.execute( "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)", (chat_id, user_id), ).fetchone()[0] def create_pending_attachment( self, attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, size_bytes: int, sha256: str, width: int, height: int, object_key: str, staging_key: Optional[str] = None, pipeline_version: int = 1, ) -> bool: """Шаг 1 из 3: намерение (state=pending) COMMIT до S3 PUT в staging. У любого объекта, который API кладёт в бакет, уже есть строка с его ключом (staging_key): объект без строки может появиться только после удаления строки сборщиком и никогда не «потеряется» навсегда (src/reconcile.py). object_key — будущий конечный ключ оригинала, его запишет worker. Проверка членства и вставка — один оператор. False -> не участник. """ with self.pool.connection() as conn: row = conn.execute( """ INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, staging_key, pipeline_version, state) SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending' WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s) RETURNING 1 """, (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, staging_key, pipeline_version, chat_id, owner_id), ).fetchone() return row is not None def accept_attachment( self, attachment_id: UUID, job_id: UUID, payload: dict, max_attempts: int, outbox_delay_seconds: float, ) -> Optional[Tuple[Attachment, int]]: """Шаг 3 из 3 (lab5): после успешного PUT в staging — ОДНА транзакция: pending -> queued, запись задания и строки outbox. COMMIT => можно 202. Возвращает (вложение, outbox_id) или None, если строку уже забрал сборщик. """ with self.pool.connection() as conn: with conn.transaction(): row = conn.cursor(row_factory=dict_row).execute( """ UPDATE attachments a SET state = 'queued', updated_at = clock_timestamp() WHERE a.id = %s AND a.state = 'pending' RETURNING {cols} """.format(cols=ATTACHMENT_COLUMNS), (attachment_id,), ).fetchone() if row is None: return None conn.execute( """ INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts) VALUES (%s, %s, %s, 'queued', 1, %s) """, (job_id, attachment_id, row["pipeline_version"], max_attempts), ) outbox_id = conn.execute( """ INSERT INTO job_outbox (job_id, payload, available_at) VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) RETURNING id """, (job_id, Jsonb(payload), outbox_delay_seconds), ).fetchone()[0] conn.execute( "INSERT INTO job_events (job_id, attachment_id, attempt, event, actor) " "VALUES (%s, %s, 1, 'accepted', 'api')", (job_id, attachment_id), ) return _attachment(row), outbox_id def mark_attachment_ready(self, attachment_id: UUID) -> Optional[Attachment]: """lab4-совместимость (синхронная загрузка без worker). В lab5 не используется.""" with self.pool.connection() as conn: row = conn.cursor(row_factory=dict_row).execute( """ UPDATE attachments a SET state = 'ready', updated_at = clock_timestamp() WHERE a.id = %s AND a.state = 'pending' RETURNING {cols} """.format(cols=ATTACHMENT_COLUMNS), (attachment_id,), ).fetchone() return _attachment(row) if row is not None else None def delete_pending_attachment(self, attachment_id: UUID) -> bool: """Компенсация при ошибке S3 PUT: удалить только своё незавершённое намерение.""" with self.pool.connection() as conn: cur = conn.execute( "DELETE FROM attachments WHERE id = %s AND state = 'pending'", (attachment_id,), ) return cur.rowcount == 1 @staticmethod def _variants(conn, attachment_id: UUID, pipeline_version: int) -> List[Variant]: rows = conn.cursor(row_factory=dict_row).execute( """ SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END """, (attachment_id, pipeline_version), ).fetchall() return [Variant(**r) for r in rows] def get_attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> Optional[Tuple[Attachment, str, int]]: """(метаданные, object_key оригинала, pipeline_version) принятого вложения, если user — участник чата. pending (приём не завершён) — как несуществующее. object_key читаем только в ready. """ with self.pool.connection() as conn: row = conn.cursor(row_factory=dict_row).execute( """ SELECT {cols}, a.object_key FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s WHERE a.id = %s AND a.state <> 'pending' """.format(cols=ATTACHMENT_COLUMNS), (user_id, attachment_id), ).fetchone() if row is None: return None variants = self._variants(conn, attachment_id, row["pipeline_version"]) if row["state"] == "ready" else [] return _attachment(row, variants), row["object_key"], row["pipeline_version"] def variant_object(self, attachment_id: UUID, pipeline_version: int, name: str) -> Optional[Tuple[str, str, int, str]]: """(object_key, content_type, size_bytes, sha256) варианта или None.""" with self.pool.connection() as conn: row = conn.execute( """ SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s AND name = %s """, (attachment_id, pipeline_version, name), ).fetchone() return tuple(row) if row else None