forked from mtusi2026/mtusi2026lab4
Добавить вложения: S3, метаданные в PostgreSQL и очистка незавершённых загрузок
This commit is contained in:
+298
@@ -0,0 +1,298 @@
|
||||
import uuid
|
||||
from datetime import timezone
|
||||
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
import config
|
||||
|
||||
|
||||
class UsernameTaken(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownMember(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class IdempotencyConflict(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownAttachment(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ширина строки фиксированная: 6 знаков микросекунд, столько же хранит timestamptz
|
||||
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
|
||||
|
||||
def timestamp(value) -> str:
|
||||
return value.astimezone(timezone.utc).strftime(TIME_FORMAT)
|
||||
|
||||
|
||||
def user_json(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def chat_json(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": [str(member) for member in row["member_ids"]],
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def message_json(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
"attachment_ids": [str(item) for item in row["attachment_ids"]],
|
||||
}
|
||||
|
||||
|
||||
def attachment_json(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": "ready",
|
||||
"variants": [],
|
||||
"error_code": None,
|
||||
"created_at": timestamp(row["created_at"]),
|
||||
"updated_at": timestamp(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
CHAT_COLUMNS = """
|
||||
c.id, c.title, c.created_at,
|
||||
(SELECT array_agg(m.user_id ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) AS member_ids
|
||||
"""
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """
|
||||
m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
COALESCE((SELECT array_agg(a.attachment_id ORDER BY a.position) FROM message_attachments a
|
||||
WHERE a.message_id = m.id), '{}'::uuid[]) AS attachment_ids
|
||||
"""
|
||||
|
||||
ATTACHMENT_COLUMNS = "a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.created_at, a.updated_at"
|
||||
|
||||
|
||||
class Store:
|
||||
"""Все данные мессенджера в PostgreSQL. Значения в запросы попадают только через параметры %s."""
|
||||
|
||||
def __init__(self, url: str):
|
||||
self.pool = ConnectionPool(
|
||||
url,
|
||||
min_size=config.DB_POOL_MIN_SIZE,
|
||||
max_size=config.DB_POOL_MAX_SIZE,
|
||||
timeout=config.DB_POOL_TIMEOUT, # сколько запрос ждёт свободное соединение
|
||||
kwargs={
|
||||
"connect_timeout": config.DB_CONNECT_TIMEOUT,
|
||||
"options": f"-c statement_timeout={config.DB_STATEMENT_TIMEOUT * 1000}",
|
||||
"row_factory": dict_row,
|
||||
},
|
||||
# перед выдачей соединение проверяется, поэтому после перезапуска БД мёртвые соединения не достаются запросам
|
||||
check=ConnectionPool.check_connection,
|
||||
open=False,
|
||||
)
|
||||
|
||||
def open(self) -> None:
|
||||
# не ждём БД: если она недоступна, сервер стартует и отвечает 503
|
||||
self.pool.open(wait=False)
|
||||
|
||||
def close(self) -> None:
|
||||
self.pool.close()
|
||||
|
||||
def ping(self) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
def create_user(self, username: str, display_name: str, password_hash: str) -> dict:
|
||||
with self.pool.connection() as conn:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (username, display_name, password_hash) VALUES (%s, %s, %s) "
|
||||
"RETURNING id, username, display_name, created_at",
|
||||
(username, display_name, password_hash),
|
||||
).fetchone()
|
||||
except errors.UniqueViolation:
|
||||
# уникальность проверяет сама БД, поэтому два одновременных запроса не создадут двух пользователей
|
||||
raise UsernameTaken()
|
||||
return user_json(row)
|
||||
|
||||
def get_user(self, user_id: str):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,)
|
||||
).fetchone()
|
||||
return user_json(row) if row else None
|
||||
|
||||
def get_credentials(self, username: str):
|
||||
"""Пользователь и hash его пароля или (None, None)."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None, None
|
||||
return user_json(row), row["password_hash"]
|
||||
|
||||
def create_chat(self, title: str, member_ids: list) -> dict:
|
||||
# блок with — одна транзакция: commit при выходе, rollback при любом исключении
|
||||
with self.pool.connection() as conn:
|
||||
chat = conn.execute(
|
||||
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", (title,)
|
||||
).fetchone()
|
||||
try:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
|
||||
[(chat["id"], member, position) for position, member in enumerate(member_ids)],
|
||||
)
|
||||
except errors.ForeignKeyViolation:
|
||||
# участника нет в users: транзакция откатывается вместе с уже вставленным чатом
|
||||
raise UnknownMember()
|
||||
return chat_json({**chat, "member_ids": member_ids})
|
||||
|
||||
def get_chat(self, chat_id: str, user_id: str):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
f"SELECT {CHAT_COLUMNS} FROM chats c "
|
||||
"JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s WHERE c.id = %s",
|
||||
(user_id, chat_id),
|
||||
).fetchone()
|
||||
return chat_json(row) if row else None
|
||||
|
||||
def list_chats(self, user_id: str, after, limit: int):
|
||||
query = f"SELECT {CHAT_COLUMNS} FROM chats c JOIN chat_members me ON me.chat_id = c.id WHERE me.user_id = %s"
|
||||
params = [user_id]
|
||||
if after:
|
||||
query += " AND (c.created_at, c.id) > (%s, %s)"
|
||||
params += after
|
||||
query += " ORDER BY c.created_at, c.id LIMIT %s"
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(query, params + [limit + 1]).fetchall()
|
||||
return [chat_json(row) for row in rows[:limit]], len(rows) > limit
|
||||
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str, attachment_ids: list, key: str):
|
||||
"""Отправляет сообщение один раз на ключ. Повтор с тем же телом возвращает исходное сообщение."""
|
||||
message_id = uuid.uuid4()
|
||||
# ключ, сообщение и ссылки на вложения пишутся в одной транзакции: при сбое не остаётся ничего
|
||||
with self.pool.connection() as conn:
|
||||
if not self._is_member(conn, chat_id, sender_id):
|
||||
return None
|
||||
# параллельный запрос с тем же ключом ждёт здесь, пока первая транзакция не завершится;
|
||||
# строка возвращается, только если ключ свободен или его срок вышел
|
||||
claimed = conn.execute(
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, key, message_id) VALUES (%s, %s, %s, %s) "
|
||||
"ON CONFLICT (user_id, chat_id, key) DO UPDATE "
|
||||
"SET message_id = EXCLUDED.message_id, created_at = now() "
|
||||
"WHERE idempotency_keys.created_at < now() - make_interval(secs => %s) "
|
||||
"RETURNING message_id",
|
||||
(sender_id, chat_id, key, message_id, config.IDEMPOTENCY_TTL_SECONDS),
|
||||
).fetchone()
|
||||
if claimed:
|
||||
self._check_attachments(conn, chat_id, attachment_ids)
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text) VALUES (%s, %s, %s, %s)",
|
||||
(message_id, chat_id, sender_id, text),
|
||||
)
|
||||
if attachment_ids:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
||||
[(message_id, item, position) for position, item in enumerate(attachment_ids)],
|
||||
)
|
||||
row = conn.execute(
|
||||
f"SELECT {MESSAGE_COLUMNS} FROM idempotency_keys k JOIN messages m ON m.id = k.message_id "
|
||||
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s",
|
||||
(sender_id, chat_id, key),
|
||||
).fetchone()
|
||||
message = message_json(row)
|
||||
if message["text"] != text or message["attachment_ids"] != attachment_ids:
|
||||
raise IdempotencyConflict()
|
||||
return message
|
||||
|
||||
@staticmethod
|
||||
def _check_attachments(conn, chat_id: str, attachment_ids: list) -> None:
|
||||
"""Все вложения должны быть готовы и лежать в этом чате; исключение откатывает транзакцию с ключом."""
|
||||
if not attachment_ids:
|
||||
return
|
||||
found = conn.execute(
|
||||
"SELECT count(*) AS total FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND state = 'ready'",
|
||||
(attachment_ids, chat_id),
|
||||
).fetchone()
|
||||
if found["total"] != len(attachment_ids):
|
||||
raise UnknownAttachment()
|
||||
|
||||
def create_pending_attachment(self, attachment_id, chat_id, owner_id, filename, content_type, size, sha256, key) -> None:
|
||||
# строка записывается до объекта: так в S3 не появится объект, о котором не знает БД
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, state) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'pending')",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size, sha256, key),
|
||||
)
|
||||
|
||||
def mark_attachment_ready(self, attachment_id):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"UPDATE attachments a SET state = 'ready', updated_at = now() "
|
||||
f"WHERE a.id = %s AND a.state = 'pending' RETURNING {ATTACHMENT_COLUMNS}",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
return attachment_json(row) if row else None
|
||||
|
||||
def get_attachment(self, attachment_id: str, user_id: str):
|
||||
"""Готовое вложение и его object_key, если пользователь состоит в чате."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
f"SELECT {ATTACHMENT_COLUMNS}, a.object_key FROM attachments a "
|
||||
"JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s "
|
||||
"WHERE a.id = %s AND a.state = 'ready'",
|
||||
(user_id, attachment_id),
|
||||
).fetchone()
|
||||
return (attachment_json(row), row["object_key"]) if row else (None, None)
|
||||
|
||||
def is_member(self, chat_id: str, user_id: str) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
return self._is_member(conn, chat_id, user_id)
|
||||
|
||||
def list_messages(self, chat_id: str, user_id: str, after, limit: int):
|
||||
query = f"SELECT {MESSAGE_COLUMNS} FROM messages m WHERE m.chat_id = %s"
|
||||
params = [chat_id]
|
||||
if after:
|
||||
# keyset: продолжаем с записи после cursor, а не пропускаем N строк
|
||||
query += " AND (m.created_at, m.id) > (%s, %s)"
|
||||
params += after
|
||||
query += " ORDER BY m.created_at, m.id LIMIT %s"
|
||||
with self.pool.connection() as conn:
|
||||
if not self._is_member(conn, chat_id, user_id):
|
||||
return None
|
||||
# limit + 1: лишняя строка показывает, есть ли следующая страница
|
||||
rows = conn.execute(query, params + [limit + 1]).fetchall()
|
||||
return [message_json(row) for row in rows[:limit]], len(rows) > limit
|
||||
|
||||
@staticmethod
|
||||
def _is_member(conn, chat_id: str, user_id: str) -> bool:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||
).fetchone()
|
||||
return row is not None
|
||||
Reference in New Issue
Block a user