Files
mtusi2026lab6-fork/src/storage.py
T

662 lines
30 KiB
Python

"""Доступ к PostgreSQL. Все запросы параметризованы (%s), SQL не склеивается
со значениями пользователя. Каждая публичная операция — одна транзакция
(`with pool.connection()` делает COMMIT при успехе и ROLLBACK при исключении).
"""
from datetime import datetime
from typing import List, Optional, Sequence, Tuple
from uuid import UUID, uuid4
from psycopg import errors
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool
from .models import Attachment, Chat, Message, User, Variant
class UsernameTaken(Exception):
pass
class UnknownMember(Exception):
pass
class NotChatMember(Exception):
"""Чата нет или пользователь не участник -> 404."""
class IdempotencyConflict(Exception):
"""Тот же ключ уже использован с другим телом -> 409."""
class IdempotencyRace(Exception):
"""Строка ключа исчезла между шагами -> 503, клиент повторяет с тем же ключом."""
class AttachmentNotFound(Exception):
"""Вложение не существует, ещё не принято (pending) или принадлежит другому чату -> 404."""
class AttachmentFailed(Exception):
"""lab5: вложение в состоянии failed нельзя прикрепить к сообщению -> 409."""
# Одна реплика за раз выполняет сборку мусора вложений (src/reconcile.py).
ATTACHMENT_GC_LOCK_ID = 2026_0004
ATTACHMENT_COLUMNS = (
"a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
"a.sha256, a.state, a.error_code, a.pipeline_version, a.created_at, a.updated_at"
)
# Внутреннее состояние -> публичный status. pending (приём ещё не завершён) наружу
# не попадает: такие строки запросы API не выбирают.
PUBLIC_STATES = ("queued", "processing", "ready", "failed")
# Подзапрос: attachment_ids сообщения в исходном порядке (position).
MESSAGE_ATTACHMENT_IDS = (
"COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) "
"FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[]) AS attachment_ids"
)
def _attachment(row, variants=()) -> Attachment:
state = row["state"]
return Attachment(
id=row["id"],
chat_id=row["chat_id"],
owner_id=row["owner_id"],
filename=row["filename"],
content_type=row["content_type"],
size_bytes=row["size_bytes"],
sha256=row["sha256"],
status=state,
# Варианты публикуются вместе со статусом ready (одна транзакция worker).
variants=list(variants) if state == "ready" else [],
error_code=row.get("error_code") if state == "failed" else None,
created_at=row["created_at"],
updated_at=row["updated_at"],
)
# Одна реплика за раз чистит истёкшие ключи идемпотентности.
IDEMPOTENCY_CLEANUP_LOCK_ID = 2026_0003
Position = Optional[Tuple[datetime, UUID]]
def _user(row) -> User:
return User(
id=row["id"],
username=row["username"],
display_name=row["display_name"],
created_at=row["created_at"],
)
class Storage:
def __init__(self, pool: ConnectionPool) -> None:
self.pool = pool
def ping(self) -> None:
with self.pool.connection() as conn:
conn.execute("SELECT 1")
# users
def create_user(self, username: str, display_name: str, password_hash: str) -> User:
try:
with self.pool.connection() as conn:
row = conn.cursor(row_factory=dict_row).execute(
"""
INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
""",
(uuid4(), username, display_name, password_hash),
).fetchone()
except errors.UniqueViolation:
# Гонку двух одинаковых регистраций решает UNIQUE(username) в БД,
# а не проверка «SELECT, потом INSERT» в приложении.
raise UsernameTaken()
return _user(row)
def get_credentials(self, username: str) -> Optional[Tuple[User, str]]:
with self.pool.connection() as conn:
row = conn.cursor(row_factory=dict_row).execute(
"""
SELECT id, username, display_name, created_at, password_hash
FROM users WHERE username = %s
""",
(username,),
).fetchone()
if row is None:
return None
return _user(row), row["password_hash"]
# sessions (источник истины)
def create_session(self, token_hash: bytes, user_id: UUID, ttl_seconds: int) -> datetime:
with self.pool.connection() as conn:
# Заодно убираем давно истёкшие сессии этого пользователя.
conn.execute(
"DELETE FROM sessions WHERE user_id = %s AND expires_at < now() - interval '1 day'",
(user_id,),
)
row = conn.execute(
"""
INSERT INTO sessions (token_hash, user_id, expires_at)
VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))
RETURNING expires_at
""",
(token_hash, user_id, ttl_seconds),
).fetchone()
return row[0]
def find_session(self, token_hash: bytes) -> Optional[Tuple[User, datetime]]:
with self.pool.connection() as conn:
row = conn.cursor(row_factory=dict_row).execute(
"""
SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
FROM sessions s JOIN users u ON u.id = s.user_id
WHERE s.token_hash = %s
AND s.revoked_at IS NULL
AND s.expires_at > clock_timestamp()
""",
(token_hash,),
).fetchone()
if row is None:
return None
return _user(row), row["expires_at"]
def revoke_session(self, token_hash: bytes) -> bool:
"""Отзыв и долг «инвалидировать кэш» фиксируются одним COMMIT."""
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"""
UPDATE sessions SET revoked_at = clock_timestamp()
WHERE token_hash = %s
AND revoked_at IS NULL
AND expires_at > clock_timestamp()
RETURNING 1
""",
(token_hash,),
).fetchone()
if row is not None:
conn.execute(
"""
INSERT INTO session_cache_invalidations (token_hash)
VALUES (%s) ON CONFLICT (token_hash) DO NOTHING
""",
(token_hash,),
)
return row is not None
# долг инвалидации кэша
def has_pending_invalidations(self) -> bool:
with self.pool.connection() as conn:
return conn.execute(
"SELECT EXISTS (SELECT 1 FROM session_cache_invalidations)"
).fetchone()[0]
def pending_invalidations(self, limit: int = 500) -> List[Tuple[bytes, float]]:
"""[(token_hash, возраст в секундах)] — самые старые первыми."""
with self.pool.connection() as conn:
rows = conn.execute(
"""
SELECT token_hash, EXTRACT(EPOCH FROM clock_timestamp() - revoked_at)
FROM session_cache_invalidations
ORDER BY revoked_at LIMIT %s
""",
(limit,),
).fetchall()
return [(bytes(h), float(age)) for h, age in rows]
def delete_invalidations(self, token_hashes: Sequence[bytes]) -> None:
if not token_hashes:
return
with self.pool.connection() as conn:
conn.execute(
"DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)",
(list(token_hashes),),
)
# chats
def _insert_members(self, conn, chat_id: UUID, member_ids: Sequence[UUID]) -> None:
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
[(chat_id, member_id, pos) for pos, member_id in enumerate(member_ids)],
)
def create_chat(self, title: str, member_ids: Sequence[UUID]) -> Chat:
"""Чат и все участники создаются в одной транзакции.
Существование участников проверяет внешний ключ chat_members.user_id:
если хоть один id неизвестен, INSERT падает, и откатывается вся
транзакция, включая уже вставленную строку chats и других участников.
"""
chat_id = uuid4()
try:
with self.pool.connection() as conn:
with conn.transaction():
created_at = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING created_at",
(chat_id, title),
).fetchone()[0]
self._insert_members(conn, chat_id, member_ids)
except errors.ForeignKeyViolation:
raise UnknownMember()
return Chat(id=chat_id, title=title, member_ids=list(member_ids), created_at=created_at)
def _members_for(self, conn, chat_ids: List[UUID]) -> dict:
members = {chat_id: [] for chat_id in chat_ids}
if not chat_ids:
return members
rows = conn.execute(
"""
SELECT chat_id, user_id FROM chat_members
WHERE chat_id = ANY(%s) ORDER BY chat_id, position
""",
(chat_ids,),
).fetchall()
for chat_id, user_id in rows:
members[chat_id].append(user_id)
return members
def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> Optional[Chat]:
with self.pool.connection() as conn:
row = conn.cursor(row_factory=dict_row).execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s
WHERE c.id = %s
""",
(user_id, chat_id),
).fetchone()
if row is None:
return None
members = self._members_for(conn, [row["id"]])
return Chat(id=row["id"], title=row["title"], member_ids=members[row["id"]], created_at=row["created_at"])
def list_user_chats(self, user_id: UUID, limit: int, after: Position) -> Tuple[List[Chat], bool]:
params: list = [user_id]
keyset = ""
if after is not None:
keyset = "AND (c.created_at, c.id) > (%s, %s)"
params.extend(after)
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.cursor(row_factory=dict_row).execute(
"""
SELECT c.id, c.title, c.created_at
FROM chat_members m JOIN chats c ON c.id = m.chat_id
WHERE m.user_id = %s {keyset}
ORDER BY c.created_at, c.id
LIMIT %s
""".format(keyset=keyset),
params,
).fetchall()
has_more = len(rows) > limit
rows = rows[:limit]
members = self._members_for(conn, [r["id"] for r in rows])
chats = [
Chat(id=r["id"], title=r["title"], member_ids=members[r["id"]], created_at=r["created_at"])
for r in rows
]
return chats, has_more
# messages
@staticmethod
def _insert_message(conn, message_id: UUID, chat_id: UUID, sender_id: UUID, text: str) -> Optional[Message]:
"""Проверка членства и вставка — один оператор, без окна гонки."""
row = conn.cursor(row_factory=dict_row).execute(
"""
INSERT INTO messages (id, chat_id, sender_id, text)
SELECT %s, %s, %s, %s
WHERE EXISTS (
SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s
)
RETURNING id, chat_id, sender_id, text, created_at
""",
(message_id, chat_id, sender_id, text, chat_id, sender_id),
).fetchone()
return Message(**row) if row is not None else None
@staticmethod
def _link_attachments(conn, message: Message, attachment_ids: Sequence[UUID]) -> None:
"""Привязка вложений в той же транзакции, что и сообщение.
lab5: можно ссылаться на queued/processing/ready вложения ЭТОГО чата.
Неизвестное/чужое/ещё не принятое (pending) -> AttachmentNotFound (404);
failed -> AttachmentFailed (409). Любое исключение -> ROLLBACK сообщения и
ключа идемпотентности. FOR KEY SHARE не даёт удалить строку до COMMIT;
составной FK (attachment_id, chat_id) дублирует проверку чата в БД.
"""
if not attachment_ids:
return
ids = list(attachment_ids)
rows = conn.execute(
"""
SELECT id, state FROM attachments
WHERE id = ANY(%s) AND chat_id = %s AND state <> 'pending'
FOR KEY SHARE
""",
(ids, message.chat_id),
).fetchall()
if len(rows) != len(ids):
raise AttachmentNotFound()
if any(state == "failed" for _id, state in rows):
raise AttachmentFailed()
with conn.cursor() as cur:
cur.executemany(
"""
INSERT INTO message_attachments (message_id, attachment_id, chat_id, position)
VALUES (%s, %s, %s, %s)
""",
[(message.id, aid, message.chat_id, pos) for pos, aid in enumerate(ids)],
)
message.attachment_ids = ids
def create_message(
self,
chat_id: UUID,
sender_id: UUID,
text: str,
idem_key: str,
request_hash: bytes,
idem_ttl_seconds: int,
attachment_ids: Sequence[UUID] = (),
) -> Tuple[Message, bool]:
"""Идемпотентная отправка. Возвращает (сообщение, replayed).
Всё — одна транзакция READ COMMITTED (см. migrations/003):
1. INSERT строки ключа. Если такой ключ уже есть и он не истёк,
ON CONFLICT ... WHERE ничего не меняет и RETURNING пуст. Если
конкурентная транзакция с тем же ключом ещё не завершилась, INSERT
ЖДЁТ её COMMIT/ROLLBACK на уникальном индексе — поэтому одновременные
повторы через разные реплики создают ровно одно сообщение.
2a. Ключ наш -> INSERT сообщения (с проверкой членства). Не участник ->
исключение -> ROLLBACK: ключ не «сгорает» на 404.
2b. Ключ чужой (уже подтверждён) -> снова проверяем доступ, сравниваем
hash тела: совпал -> то же сообщение; нет -> 409.
Ошибка/обрыв на любом шаге откатывает и ключ, и сообщение.
"""
new_id = uuid4()
with self.pool.connection() as conn:
with conn.transaction():
owned = conn.execute(
"""
INSERT INTO idempotency_keys
(user_id, method, chat_id, idem_key, request_hash, message_id, expires_at)
VALUES (%s, 'POST', %s, %s, %s, %s,
clock_timestamp() + make_interval(secs => %s))
ON CONFLICT (user_id, method, chat_id, idem_key) DO UPDATE
SET request_hash = EXCLUDED.request_hash,
message_id = EXCLUDED.message_id,
created_at = clock_timestamp(),
expires_at = EXCLUDED.expires_at
-- истёкший (ещё не удалённый очисткой) ключ можно занять заново
WHERE idempotency_keys.expires_at <= clock_timestamp()
RETURNING message_id
""",
(sender_id, chat_id, idem_key, request_hash, new_id, idem_ttl_seconds),
).fetchone()
if owned is not None:
message = self._insert_message(conn, new_id, chat_id, sender_id, text)
if message is None:
raise NotChatMember() # -> ROLLBACK вместе со строкой ключа
# lab4: чужое/неготовое вложение -> исключение -> ROLLBACK всего.
self._link_attachments(conn, message, attachment_ids)
return message, False
# Ключ уже подтверждён другой (возможно, конкурентной) транзакцией.
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
).fetchone()
if member is None:
raise NotChatMember()
row = conn.cursor(row_factory=dict_row).execute(
"""
SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at,
{attachments}
FROM idempotency_keys k
JOIN messages m ON m.id = k.message_id
WHERE k.user_id = %s AND k.method = 'POST'
AND k.chat_id = %s AND k.idem_key = %s
""".format(attachments=MESSAGE_ATTACHMENT_IDS),
(sender_id, chat_id, idem_key),
).fetchone()
if row is None:
# Теоретически: строка удалена между шагами (очистка/каскад).
# Отдаём 503-подобную транзиентную ошибку, клиент повторит с тем же ключом.
raise IdempotencyRace()
if bytes(row.pop("request_hash")) != request_hash:
raise IdempotencyConflict()
return Message(**row), True
def delete_expired_idempotency_keys(self, batch: int = 1000) -> Optional[int]:
"""Удаляет истёкшие ключи пачкой. None — очистку сейчас выполняет другая реплика."""
with self.pool.connection() as conn:
with conn.transaction():
got = conn.execute(
"SELECT pg_try_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,)
).fetchone()[0]
if not got:
return None
cur = conn.execute(
"""
DELETE FROM idempotency_keys
WHERE ctid IN (
SELECT ctid FROM idempotency_keys
WHERE expires_at <= clock_timestamp()
LIMIT %s
)
""",
(batch,),
)
return cur.rowcount
def list_messages(
self, chat_id: UUID, user_id: UUID, limit: int, after: Position
) -> Optional[Tuple[List[Message], bool]]:
params: list = [chat_id]
keyset = ""
if after is not None:
keyset = "AND (created_at, id) > (%s, %s)"
params.extend(after)
params.append(limit + 1)
with self.pool.connection() as conn:
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if member is None:
return None
# Keyset-пагинация по индексу messages_chat_created_id_idx:
# Index Scan сразу с позиции курсора, читается limit+1 строк.
rows = conn.cursor(row_factory=dict_row).execute(
"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
{attachments}
FROM messages m
WHERE m.chat_id = %s {keyset}
ORDER BY m.created_at, m.id
LIMIT %s
""".format(keyset=keyset.replace("(created_at, id)", "(m.created_at, m.id)"),
attachments=MESSAGE_ATTACHMENT_IDS),
params,
).fetchall()
has_more = len(rows) > limit
return [Message(**r) for r in rows[:limit]], has_more
# attachments (lab4)
def is_chat_member(self, chat_id: UUID, user_id: UUID) -> bool:
with self.pool.connection() as conn:
return conn.execute(
"SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)",
(chat_id, user_id),
).fetchone()[0]
def create_pending_attachment(
self,
attachment_id: UUID,
chat_id: UUID,
owner_id: UUID,
filename: str,
content_type: str,
size_bytes: int,
sha256: str,
width: int,
height: int,
object_key: str,
staging_key: Optional[str] = None,
pipeline_version: int = 1,
) -> bool:
"""Шаг 1 из 3: намерение (state=pending) COMMIT до S3 PUT в staging.
У любого объекта, который API кладёт в бакет, уже есть строка с его ключом
(staging_key): объект без строки может появиться только после удаления
строки сборщиком и никогда не «потеряется» навсегда (src/reconcile.py).
object_key — будущий конечный ключ оригинала, его запишет worker.
Проверка членства и вставка — один оператор. False -> не участник.
"""
with self.pool.connection() as conn:
row = conn.execute(
"""
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type,
size_bytes, sha256, width, height, object_key,
staging_key, pipeline_version, state)
SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending'
WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)
RETURNING 1
""",
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, width, height, object_key, staging_key, pipeline_version, chat_id, owner_id),
).fetchone()
return row is not None
def accept_attachment(
self,
attachment_id: UUID,
job_id: UUID,
payload: dict,
max_attempts: int,
outbox_delay_seconds: float,
) -> Optional[Tuple[Attachment, int]]:
"""Шаг 3 из 3 (lab5): после успешного PUT в staging — ОДНА транзакция:
pending -> queued, запись задания и строки outbox. COMMIT => можно 202.
Возвращает (вложение, outbox_id) или None, если строку уже забрал сборщик.
"""
with self.pool.connection() as conn:
with conn.transaction():
row = conn.cursor(row_factory=dict_row).execute(
"""
UPDATE attachments a SET state = 'queued', updated_at = clock_timestamp()
WHERE a.id = %s AND a.state = 'pending'
RETURNING {cols}
""".format(cols=ATTACHMENT_COLUMNS),
(attachment_id,),
).fetchone()
if row is None:
return None
conn.execute(
"""
INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status,
attempt, max_attempts)
VALUES (%s, %s, %s, 'queued', 1, %s)
""",
(job_id, attachment_id, row["pipeline_version"], max_attempts),
)
outbox_id = conn.execute(
"""
INSERT INTO job_outbox (job_id, payload, available_at)
VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))
RETURNING id
""",
(job_id, Jsonb(payload), outbox_delay_seconds),
).fetchone()[0]
conn.execute(
"INSERT INTO job_events (job_id, attachment_id, attempt, event, actor) "
"VALUES (%s, %s, 1, 'accepted', 'api')",
(job_id, attachment_id),
)
return _attachment(row), outbox_id
def mark_attachment_ready(self, attachment_id: UUID) -> Optional[Attachment]:
"""lab4-совместимость (синхронная загрузка без worker). В lab5 не используется."""
with self.pool.connection() as conn:
row = conn.cursor(row_factory=dict_row).execute(
"""
UPDATE attachments a SET state = 'ready', updated_at = clock_timestamp()
WHERE a.id = %s AND a.state = 'pending'
RETURNING {cols}
""".format(cols=ATTACHMENT_COLUMNS),
(attachment_id,),
).fetchone()
return _attachment(row) if row is not None else None
def delete_pending_attachment(self, attachment_id: UUID) -> bool:
"""Компенсация при ошибке S3 PUT: удалить только своё незавершённое намерение."""
with self.pool.connection() as conn:
cur = conn.execute(
"DELETE FROM attachments WHERE id = %s AND state = 'pending'",
(attachment_id,),
)
return cur.rowcount == 1
@staticmethod
def _variants(conn, attachment_id: UUID, pipeline_version: int) -> List[Variant]:
rows = conn.cursor(row_factory=dict_row).execute(
"""
SELECT name, content_type, size_bytes, width, height, sha256
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s
ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END
""",
(attachment_id, pipeline_version),
).fetchall()
return [Variant(**r) for r in rows]
def get_attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> Optional[Tuple[Attachment, str, int]]:
"""(метаданные, object_key оригинала, pipeline_version) принятого вложения, если user — участник чата.
pending (приём не завершён) — как несуществующее. object_key читаем только в ready.
"""
with self.pool.connection() as conn:
row = conn.cursor(row_factory=dict_row).execute(
"""
SELECT {cols}, a.object_key
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s
WHERE a.id = %s AND a.state <> 'pending'
""".format(cols=ATTACHMENT_COLUMNS),
(user_id, attachment_id),
).fetchone()
if row is None:
return None
variants = self._variants(conn, attachment_id, row["pipeline_version"]) if row["state"] == "ready" else []
return _attachment(row, variants), row["object_key"], row["pipeline_version"]
def variant_object(self, attachment_id: UUID, pipeline_version: int, name: str) -> Optional[Tuple[str, str, int, str]]:
"""(object_key, content_type, size_bytes, sha256) варианта или None."""
with self.pool.connection() as conn:
row = conn.execute(
"""
SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s AND name = %s
""",
(attachment_id, pipeline_version, name),
).fetchone()
return tuple(row) if row else None