Files
mtusi2026lab4/app/repo.py
T
alenchha 078a8341b5 ЛР 4: вложения в S3 (MinIO), reconciler БД<->S3, отчёт и evidence
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API
с потоковой проверкой целостности, метаданные в PostgreSQL и байты в
приватном bucket, отдельный S3-пользователь с ограниченными правами,
идемпотентный bootstrap bucket, компенсация и reconciler (трек 2),
тесты отказов, эксперименты по памяти и согласованности.
2026-11-02 16:59:17 +03:00

294 lines
10 KiB
Python

import hashlib
import json
from typing import List, Optional, Tuple
import psycopg
from app.cursor import fmt
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
CHAT_QUERY = (
"SELECT c.id, c.title, c.created_at,"
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
)
class UnknownMember(Exception):
pass
def decoded(value) -> str:
return bytes(value).decode("utf-8")
def public_user(row) -> dict:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": decoded(row["display_name"]),
"created_at": fmt(row["created_at"]),
}
def _chat(row) -> dict:
return {
"id": str(row["id"]),
"title": decoded(row["title"]),
"member_ids": [str(value) for value in row["member_ids"]],
"created_at": fmt(row["created_at"]),
}
def _message(row) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": decoded(row["text"]),
"created_at": fmt(row["created_at"]),
}
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
msg = _message(row)
msg["attachment_ids"] = attachment_ids
return msg
def _attachment(row) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [],
"error_code": row["error_code"],
"created_at": fmt(row["created_at"]),
"updated_at": fmt(row["updated_at"]),
}
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
try:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
" RETURNING id, username, display_name, created_at",
(username, display_name.encode("utf-8"), password_hash),
)
except psycopg.errors.UniqueViolation:
return None
return public_user(await cur.fetchone())
async def credentials(conn, username: str):
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,),
)
return await cur.fetchone()
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_at)"
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
" RETURNING id, title, created_at",
(title.encode("utf-8"),),
)
chat = await cur.fetchone()
cur = await conn.execute(
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
(chat["id"], member_ids),
)
if cur.rowcount != len(member_ids):
raise UnknownMember()
chat["member_ids"] = list(member_ids)
return _chat(chat)
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
cur = await conn.execute(
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id),
)
row = await cur.fetchone()
return _chat(row) if row else None
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
CHAT_QUERY
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
" ORDER BY c.created_at, c.id LIMIT %s",
(user_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
return [_chat(row) for row in rows[:limit]], len(rows) > limit
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
)
row = await cur.fetchone()
return _message(row) if row else None
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
cur = await conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
(message_id,),
)
rows = await cur.fetchall()
return [str(row["attachment_id"]) for row in rows]
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
for pos, aid in enumerate(attachment_ids):
await conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message_id, aid, pos),
)
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
" ORDER BY created_at, id LIMIT %s",
(chat_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
messages = []
for row in rows[:limit]:
aids = await _fetch_message_attachment_ids(conn, row["id"])
messages.append(_message_with_attachments(row, aids))
return messages, len(rows) > limit
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
raw = "%s:%s:%s" % (user_id, chat_id, key)
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
attachment_ids: List[str], idem_key: str, body_hash: str):
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
)
if await cur.fetchone() is None:
return "not_member", None
cur = await conn.execute(
"SELECT request_body_hash, response_json FROM idempotency_keys"
" WHERE user_id = %s AND chat_id = %s AND key = %s",
(sender_id, chat_id, idem_key),
)
existing = await cur.fetchone()
if existing:
if existing["request_body_hash"] != body_hash:
return "conflict", None
return "replay", json.loads(existing["response_json"])
if attachment_ids:
cur = await conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
(attachment_ids,),
)
att_rows = await cur.fetchall()
found_ids = {str(row["id"]) for row in att_rows}
for aid in attachment_ids:
if aid not in found_ids:
return "attachment_not_found", None
for row in att_rows:
if str(row["chat_id"]) != chat_id:
return "attachment_not_found", None
if row["status"] == "failed":
return "attachment_failed", None
text_bytes = text.encode("utf-8") if text else b""
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text_bytes, chat_id, sender_id),
)
row = await cur.fetchone()
if row is None:
return "not_member", None
if attachment_ids:
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
message = _message_with_attachments(row, list(attachment_ids))
await conn.execute(
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
)
return "created", message
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
(ttl_seconds,),
)
return cur.rowcount
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
content_type: str, size_bytes: int, sha256: str, object_key: str) -> None:
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
" status, error_code, created_at, updated_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, clock_timestamp(), clock_timestamp())",
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key),
)
async def finish_attachment(conn, attachment_id: str) -> Optional[dict]:
cur = await conn.execute(
"UPDATE attachments SET status = 'ready', updated_at = clock_timestamp()"
" WHERE id = %s AND status = 'uploading'"
" RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code,"
" created_at, updated_at",
(attachment_id,),
)
row = await cur.fetchone()
return _attachment(row) if row else None
async def abort_attachment(conn, attachment_id: str) -> None:
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
async def get_attachment(conn, attachment_id: str):
cur = await conn.execute(
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, error_code,"
" created_at, updated_at FROM attachments WHERE id = %s AND status <> 'uploading'",
(attachment_id,),
)
return await cur.fetchone()