forked from mtusi2026/mtusi2026lab4
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
294 lines
10 KiB
Python
294 lines
10 KiB
Python
import hashlib
|
|
import json
|
|
from typing import List, Optional, Tuple
|
|
|
|
import psycopg
|
|
|
|
from app.cursor import fmt
|
|
|
|
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
|
|
CHAT_QUERY = (
|
|
"SELECT c.id, c.title, c.created_at,"
|
|
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
|
|
)
|
|
|
|
|
|
class UnknownMember(Exception):
|
|
pass
|
|
|
|
|
|
def decoded(value) -> str:
|
|
return bytes(value).decode("utf-8")
|
|
|
|
|
|
def public_user(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"username": row["username"],
|
|
"display_name": decoded(row["display_name"]),
|
|
"created_at": fmt(row["created_at"]),
|
|
}
|
|
|
|
|
|
def _chat(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"title": decoded(row["title"]),
|
|
"member_ids": [str(value) for value in row["member_ids"]],
|
|
"created_at": fmt(row["created_at"]),
|
|
}
|
|
|
|
|
|
def _message(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"chat_id": str(row["chat_id"]),
|
|
"sender_id": str(row["sender_id"]),
|
|
"text": decoded(row["text"]),
|
|
"created_at": fmt(row["created_at"]),
|
|
}
|
|
|
|
|
|
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
|
|
msg = _message(row)
|
|
msg["attachment_ids"] = attachment_ids
|
|
return msg
|
|
|
|
|
|
def _attachment(row) -> dict:
|
|
return {
|
|
"id": str(row["id"]),
|
|
"chat_id": str(row["chat_id"]),
|
|
"owner_id": str(row["owner_id"]),
|
|
"filename": row["filename"],
|
|
"content_type": row["content_type"],
|
|
"size_bytes": row["size_bytes"],
|
|
"sha256": row["sha256"],
|
|
"status": row["status"],
|
|
"variants": [],
|
|
"error_code": row["error_code"],
|
|
"created_at": fmt(row["created_at"]),
|
|
"updated_at": fmt(row["updated_at"]),
|
|
}
|
|
|
|
|
|
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
|
|
try:
|
|
cur = await conn.execute(
|
|
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
|
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
|
|
" RETURNING id, username, display_name, created_at",
|
|
(username, display_name.encode("utf-8"), password_hash),
|
|
)
|
|
except psycopg.errors.UniqueViolation:
|
|
return None
|
|
return public_user(await cur.fetchone())
|
|
|
|
|
|
async def credentials(conn, username: str):
|
|
cur = await conn.execute(
|
|
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
|
(username,),
|
|
)
|
|
return await cur.fetchone()
|
|
|
|
|
|
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
|
|
cur = await conn.execute(
|
|
"INSERT INTO chats (id, title, created_at)"
|
|
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
|
|
" RETURNING id, title, created_at",
|
|
(title.encode("utf-8"),),
|
|
)
|
|
chat = await cur.fetchone()
|
|
cur = await conn.execute(
|
|
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
|
|
(chat["id"], member_ids),
|
|
)
|
|
if cur.rowcount != len(member_ids):
|
|
raise UnknownMember()
|
|
chat["member_ids"] = list(member_ids)
|
|
return _chat(chat)
|
|
|
|
|
|
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
|
|
cur = await conn.execute(
|
|
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
|
|
(user_id, chat_id),
|
|
)
|
|
row = await cur.fetchone()
|
|
return _chat(row) if row else None
|
|
|
|
|
|
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
|
ts, last_id = after if after else (None, None)
|
|
cur = await conn.execute(
|
|
CHAT_QUERY
|
|
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
|
|
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
|
|
" ORDER BY c.created_at, c.id LIMIT %s",
|
|
(user_id, ts, ts, last_id, limit + 1),
|
|
)
|
|
rows = await cur.fetchall()
|
|
return [_chat(row) for row in rows[:limit]], len(rows) > limit
|
|
|
|
|
|
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
|
|
cur = await conn.execute(
|
|
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
|
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
|
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
|
" RETURNING " + MESSAGE_COLUMNS,
|
|
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
|
|
)
|
|
row = await cur.fetchone()
|
|
return _message(row) if row else None
|
|
|
|
|
|
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
|
|
cur = await conn.execute(
|
|
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
|
|
(message_id,),
|
|
)
|
|
rows = await cur.fetchall()
|
|
return [str(row["attachment_id"]) for row in rows]
|
|
|
|
|
|
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
|
|
for pos, aid in enumerate(attachment_ids):
|
|
await conn.execute(
|
|
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
|
(message_id, aid, pos),
|
|
)
|
|
|
|
|
|
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
|
ts, last_id = after if after else (None, None)
|
|
cur = await conn.execute(
|
|
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
|
|
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
|
|
" ORDER BY created_at, id LIMIT %s",
|
|
(chat_id, ts, ts, last_id, limit + 1),
|
|
)
|
|
rows = await cur.fetchall()
|
|
messages = []
|
|
for row in rows[:limit]:
|
|
aids = await _fetch_message_attachment_ids(conn, row["id"])
|
|
messages.append(_message_with_attachments(row, aids))
|
|
return messages, len(rows) > limit
|
|
|
|
|
|
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
|
|
raw = "%s:%s:%s" % (user_id, chat_id, key)
|
|
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
|
|
|
|
|
|
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
|
|
attachment_ids: List[str], idem_key: str, body_hash: str):
|
|
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
|
|
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
|
|
|
|
cur = await conn.execute(
|
|
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
|
(chat_id, sender_id),
|
|
)
|
|
if await cur.fetchone() is None:
|
|
return "not_member", None
|
|
|
|
cur = await conn.execute(
|
|
"SELECT request_body_hash, response_json FROM idempotency_keys"
|
|
" WHERE user_id = %s AND chat_id = %s AND key = %s",
|
|
(sender_id, chat_id, idem_key),
|
|
)
|
|
existing = await cur.fetchone()
|
|
|
|
if existing:
|
|
if existing["request_body_hash"] != body_hash:
|
|
return "conflict", None
|
|
return "replay", json.loads(existing["response_json"])
|
|
|
|
if attachment_ids:
|
|
cur = await conn.execute(
|
|
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
|
|
(attachment_ids,),
|
|
)
|
|
att_rows = await cur.fetchall()
|
|
found_ids = {str(row["id"]) for row in att_rows}
|
|
for aid in attachment_ids:
|
|
if aid not in found_ids:
|
|
return "attachment_not_found", None
|
|
for row in att_rows:
|
|
if str(row["chat_id"]) != chat_id:
|
|
return "attachment_not_found", None
|
|
if row["status"] == "failed":
|
|
return "attachment_failed", None
|
|
|
|
text_bytes = text.encode("utf-8") if text else b""
|
|
|
|
cur = await conn.execute(
|
|
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
|
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
|
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
|
" RETURNING " + MESSAGE_COLUMNS,
|
|
(chat_id, sender_id, text_bytes, chat_id, sender_id),
|
|
)
|
|
row = await cur.fetchone()
|
|
if row is None:
|
|
return "not_member", None
|
|
|
|
if attachment_ids:
|
|
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
|
|
|
|
message = _message_with_attachments(row, list(attachment_ids))
|
|
|
|
await conn.execute(
|
|
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
|
|
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
|
|
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
|
|
)
|
|
|
|
return "created", message
|
|
|
|
|
|
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
|
|
cur = await conn.execute(
|
|
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
|
|
(ttl_seconds,),
|
|
)
|
|
return cur.rowcount
|
|
|
|
|
|
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
|
|
content_type: str, size_bytes: int, sha256: str, object_key: str) -> None:
|
|
await conn.execute(
|
|
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
|
|
" status, error_code, created_at, updated_at)"
|
|
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, clock_timestamp(), clock_timestamp())",
|
|
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key),
|
|
)
|
|
|
|
|
|
async def finish_attachment(conn, attachment_id: str) -> Optional[dict]:
|
|
cur = await conn.execute(
|
|
"UPDATE attachments SET status = 'ready', updated_at = clock_timestamp()"
|
|
" WHERE id = %s AND status = 'uploading'"
|
|
" RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code,"
|
|
" created_at, updated_at",
|
|
(attachment_id,),
|
|
)
|
|
row = await cur.fetchone()
|
|
return _attachment(row) if row else None
|
|
|
|
|
|
async def abort_attachment(conn, attachment_id: str) -> None:
|
|
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
|
|
|
|
|
|
async def get_attachment(conn, attachment_id: str):
|
|
cur = await conn.execute(
|
|
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, error_code,"
|
|
" created_at, updated_at FROM attachments WHERE id = %s AND status <> 'uploading'",
|
|
(attachment_id,),
|
|
)
|
|
return await cur.fetchone()
|