Обновила storage/postgres.py и attachments.py

This commit is contained in:
vasyukova
2026-10-23 00:07:06 +03:00
parent 7d39c2dab6
commit 8030c8821a
2 changed files with 228 additions and 106 deletions
+48 -32
View File
@@ -1,4 +1,4 @@
"""Вложения (lab4): синхронный binary upload PNG/JPEG, авторизованный download."""
"""Вложения (lab5): upload → 202 queued, async worker, download вариантов."""
import hashlib
import io
import os
@@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import Response
from PIL import Image, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from botocore.exceptions import ClientError
from ..models.schemas import Attachment, User
from ..storage import s3
@@ -22,6 +23,7 @@ router = APIRouter(prefix="/api/v1", tags=["attachments"])
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
PIPELINE_VERSION = 1
def _require_member(chat_id: UUID, user_id: UUID):
@@ -52,11 +54,7 @@ def _detect_content_type(data: bytes) -> Optional[str]:
def _validate_image(data: bytes) -> str:
"""MIME по содержимому + лимит пикселей + полная проверка декодирования.
Полное декодирование (load) выполняется через Pillow, чтобы отвергнуть
повреждённые/обрезанные файлы, которые open() принимает по заголовку.
"""
"""MIME по содержимому + лимит пикселей + полная проверка декодирования."""
ctype = _detect_content_type(data)
if ctype is None:
raise HTTPException(status_code=415, detail="unsupported_media_type")
@@ -67,14 +65,10 @@ def _validate_image(data: bytes) -> str:
raise HTTPException(status_code=415, detail="unsupported_media_type")
if width * height > MAX_IMAGE_PIXELS:
raise HTTPException(status_code=413, detail="payload_too_large")
# Полное декодирование всех данных: битый IDAT / обрезанный файл
# поднимут OSError/ValueError и будут отвергнуты как 415.
img.load()
except HTTPException:
raise
except Image.DecompressionBombError:
# Pillow защищается от «бомб» по собственному порогу: для нас
# это превышение лимита пикселей → 413.
raise HTTPException(status_code=413, detail="payload_too_large")
except Exception:
raise HTTPException(status_code=415, detail="unsupported_media_type")
@@ -83,9 +77,9 @@ def _validate_image(data: bytes) -> str:
@router.post(
path="/chats/{chat_id}/attachments",
summary="Принять один PNG/JPEG, до 10 MiB",
summary="Принять один PNG/JPEG, до 10 MiB (lab5: 202 queued)",
response_model=Attachment,
status_code=201,
status_code=202,
)
async def upload_attachment(
chat_id: UUID,
@@ -96,7 +90,6 @@ async def upload_attachment(
_require_member(chat_id, user.id)
safe_name = _validate_filename(filename)
# Content-Length — быстрая отсечка; не доверяем полностью.
cl = request.headers.get("content-length")
if cl is not None:
try:
@@ -105,7 +98,6 @@ async def upload_attachment(
except ValueError:
raise HTTPException(status_code=400, detail="invalid_request")
# Потоковое чтение с жёстким лимитом.
chunks: list[bytes] = []
total = 0
async for chunk in request.stream():
@@ -119,16 +111,15 @@ async def upload_attachment(
if not body:
raise HTTPException(status_code=400, detail="invalid_request")
# Pillow — в threadpool, чтобы не блокировать event loop.
content_type = await run_in_threadpool(_validate_image, body)
sha256 = hashlib.sha256(body).hexdigest()
attachment_id = uuid4()
object_key = s3.original_key(chat_id, attachment_id)
staging_key = s3.staging_key(chat_id, attachment_id)
# 1) Сначала сохраняем в S3. Если следующий шаг упадёт — компенсируем.
# 1) Durable staging в S3 (приватный prefix).
try:
await run_in_threadpool(s3.put_bytes, object_key, body, content_type)
await run_in_threadpool(s3.put_bytes, staging_key, body, content_type)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=503, detail="unavailable") from exc
@@ -141,19 +132,31 @@ async def upload_attachment(
content_type=content_type,
size_bytes=len(body),
sha256=sha256,
status="ready",
status="queued",
variants=[],
error_code=None,
created_at=now,
updated_at=now,
)
# 2) INSERT в БД. При ЛЮБОМ исключении удаляем S3-объект (компенсация).
job_id = uuid4()
outbox_payload = {
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": PIPELINE_VERSION,
"created_at": now.isoformat().replace("+00:00", "Z"),
}
# 2) INSERT attachment + outbox в ОДНОЙ транзакции.
# При сбое — компенсация: удаляем staging.
try:
await run_in_threadpool(storage.create_attachment, att, object_key)
await run_in_threadpool(
storage.create_attachment_queued, att, staging_key, outbox_payload
)
except Exception:
try:
await run_in_threadpool(s3.delete_object, object_key)
await run_in_threadpool(s3.delete_object, staging_key)
except Exception:
pass
raise HTTPException(status_code=503, detail="unavailable")
@@ -180,15 +183,16 @@ def get_attachment(
@router.get(
path="/attachments/{attachment_id}/content",
summary="Скачать байты через авторизованный API",
summary="Скачать байты (original или variant) через авторизованный API",
)
def download_attachment(
attachment_id: UUID,
user: User = Depends(current_user),
variant: str = Query("original"),
):
if variant != "original":
raise HTTPException(status_code=404, detail="not_found")
if variant not in ("original", "thumbnail", "crop", "optimized"):
raise HTTPException(status_code=400, detail="invalid_request")
pair = storage.get_attachment_with_key(attachment_id)
if pair is None:
raise HTTPException(status_code=404, detail="not_found")
@@ -197,16 +201,28 @@ def download_attachment(
if att.status != "ready":
raise HTTPException(status_code=409, detail="conflict")
try:
data, ctype = s3.get_object(object_key)
except Exception as exc: # noqa: BLE001
# Объект отсутствует при готовых метаданных: 404.
if exc.__class__.__name__ == "NoSuchKey":
if variant == "original":
key = object_key
ctype_expected = att.content_type
else:
vs = [v for v in att.variants if v.name == variant]
if not vs:
raise HTTPException(status_code=404, detail="not_found")
v = vs[0]
key = s3.variant_key(att.chat_id, att.id, variant)
ctype_expected = v.content_type
try:
data, ctype = s3.get_object(key)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code", "")
if code in ("404", "NoSuchKey", "NotFound"):
raise HTTPException(status_code=404, detail="not_found")
raise HTTPException(status_code=503, detail="unavailable") from exc
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=503, detail="unavailable") from exc
return Response(
content=data,
media_type=ctype,
headers={"Content-Length": str(len(data))},
media_type=ctype or ctype_expected,
)
+180 -74
View File
@@ -1,8 +1,9 @@
import hashlib
import hmac
import json as _json
import secrets
import threading
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from typing import List, Optional
from uuid import UUID, uuid4
@@ -70,13 +71,14 @@ def _row_to_message(row, attachment_ids: List[UUID]) -> Message:
)
def _row_to_attachment(row) -> Attachment:
def _row_to_attachment(row, variants: Optional[list] = None) -> Attachment:
# row: id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
# status, error_code, created_at, updated_at
return Attachment(
id=row[0], chat_id=row[1], owner_id=row[2],
filename=row[3], content_type=row[4], size_bytes=row[5],
sha256=row[6], status=row[7], variants=[], error_code=row[8],
sha256=row[6], status=row[7], variants=variants or [],
error_code=row[8],
created_at=row[9], updated_at=row[10],
)
@@ -91,8 +93,6 @@ class PostgresStorage:
def create_user(self, username: str, display_name: str, password: str) -> User:
with self._lock, connection() as conn:
with conn.cursor() as cur:
# SELECT-проверка + INSERT в одной транзакции; уникальный индекс
# users.username отсекает гонку на уровне БД (UniqueViolation).
cur.execute("SELECT 1 FROM users WHERE username = %s", (username,))
if cur.fetchone() is not None:
raise ValueError("username_taken")
@@ -108,8 +108,6 @@ class PostgresStorage:
_hash_password(password), user.created_at),
)
except Exception as exc:
# Параллельная регистрация того же username: уникальный индекс
# выбросил UniqueViolation — превращаем в доменную ошибку.
if exc.__class__.__name__ == "UniqueViolation":
raise ValueError("username_taken")
raise
@@ -148,9 +146,7 @@ class PostgresStorage:
return None
return _row_to_user(row[:4])
# ---------- Сессии (Redis или PostgreSQL — см. storage/sessions.py) ----------
# Тонкие делегаты: роутеры продолжают вызывать storage.create_session(...),
# storage.get_session_user(...), storage.revoke_session(...) без изменений.
# ---------- Сессии ----------
def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple:
return _sess_create(user_id, ttl_seconds)
@@ -221,10 +217,7 @@ class PostgresStorage:
return chats
# ---------- Сообщения ----------
def _load_message_attachments(
self, cur, message_ids: List[UUID]
) -> dict[UUID, List[UUID]]:
"""Одним запросом достаём attachment_ids для набора сообщений."""
def _load_message_attachments(self, cur, message_ids: List[UUID]) -> dict[UUID, List[UUID]]:
if not message_ids:
return {}
cur.execute(
@@ -238,17 +231,8 @@ class PostgresStorage:
return result
def list_messages_page(
self,
chat_id: UUID,
limit: int,
after: Optional[tuple] = None,
self, chat_id: UUID, limit: int, after: Optional[tuple] = None,
) -> List[Message]:
"""Keyset-пагинация в SQL
after = (created_at, id) последней записи предыдущей страницы или None.
Запрос использует индекс idx_messages_chat_created (chat_id, created_at, id)
и не делает полный scan всей истории на каждую страницу.
"""
with connection() as conn, conn.cursor() as cur:
if after is None:
cur.execute(
@@ -282,25 +266,8 @@ class PostgresStorage:
key: str,
body_hash: str,
) -> tuple[str, Optional[Message], Optional[dict]]:
"""Атомарная отправка сообщения с идемпотентностью.
Возвращает одно из:
('created', message, None) — мы выиграли гонку и создали сообщение;
('replay', None, info) — ключ уже занят; info = {body_hash,
response_status, response_body}
('retry', None, None) — конфликт при INSERT, но строки нет
(крайне редко); вызывающий может
повторить запрос.
ВСЁ делается в ОДНОЙ транзакции: либо строка idempotency_keys
и строка messages фиксируются вместе, либо не фиксируется ничего.
Это устраняет блокер, когда сообщение сохранено, а ключ остался
с response_status=0 и повтор зависал.
"""
import json as _json
with connection() as conn:
with conn.cursor() as cur:
# 1) Пытаемся занять ключ. ON CONFLICT DO NOTHING — атомарно.
cur.execute(
"INSERT INTO idempotency_keys"
" (user_id, method, path, key, body_hash, response_status, response_body)"
@@ -312,7 +279,6 @@ class PostgresStorage:
won = cur.fetchone() is not None
if won:
# 2) Мы выиграли: создаём сообщение в ТОЙ ЖЕ транзакции.
message = Message(
id=uuid4(), chat_id=chat_id, sender_id=sender_id,
text=text, created_at=now_utc(),
@@ -332,7 +298,6 @@ class PostgresStorage:
[(message.id, aid, idx)
for idx, aid in enumerate(attachment_ids)],
)
# 3) Пишем финальный ответ в ту же строку ключа.
body_json = _json.loads(message.model_dump_json())
cur.execute(
"UPDATE idempotency_keys"
@@ -341,11 +306,9 @@ class PostgresStorage:
(201, _json.dumps(body_json),
user_id, method, path, key),
)
# Один commit — либо всё, либо ничего.
conn.commit()
return ("created", message, None)
# Ключ уже занят. Читаем строку в этой же транзакции.
cur.execute(
"SELECT body_hash, response_status, response_body"
" FROM idempotency_keys"
@@ -356,7 +319,6 @@ class PostgresStorage:
conn.commit()
if row is None:
# Только что был конфликт, а строки нет — редкий случай; повтор.
return ("retry", None, None)
return (
"replay",
@@ -369,11 +331,8 @@ class PostgresStorage:
)
# ---------- Вложения ----------
def create_attachment(
self,
attachment: Attachment,
object_key: str,
) -> Attachment:
def create_attachment(self, attachment: Attachment, object_key: str) -> Attachment:
"""lab4 fallback (синхронный ready). Не используется в lab5."""
with connection() as conn, conn.cursor() as cur:
cur.execute(
"INSERT INTO attachments"
@@ -391,27 +350,112 @@ class PostgresStorage:
conn.commit()
return attachment
def create_attachment_queued(
self,
attachment: Attachment,
staging_key: str,
outbox_payload: dict,
) -> Attachment:
"""lab5: attachment в queued + строка outbox в ОДНОЙ транзакции."""
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO attachments"
" (id, chat_id, owner_id, filename, content_type, size_bytes,"
" sha256, object_key, status, error_code, processing_until,"
" created_at, updated_at)"
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
attachment.id, attachment.chat_id, attachment.owner_id,
attachment.filename, attachment.content_type,
attachment.size_bytes, attachment.sha256, staging_key,
"queued", None, None,
attachment.created_at, attachment.updated_at,
),
)
cur.execute(
"INSERT INTO outbox (job_id, payload) VALUES (%s, %s)",
(outbox_payload["job_id"], _json.dumps(outbox_payload)),
)
conn.commit()
return attachment
def claim_processing(self, attachment_id: UUID, lease_seconds: int = 60) -> bool:
with connection() as conn:
with conn.cursor() as cur:
now = now_utc()
cur.execute(
"UPDATE attachments SET status='processing',"
" processing_until=%s, updated_at=%s"
" WHERE id=%s AND status IN ('queued','processing')"
" RETURNING id",
(now + timedelta(seconds=lease_seconds), now, attachment_id),
)
won = cur.fetchone() is not None
conn.commit()
return won
def recover_stuck_processing(self) -> int:
with connection() as conn:
with conn.cursor() as cur:
now = now_utc()
cur.execute(
"UPDATE attachments SET status='queued', processing_until=NULL,"
" updated_at=%s"
" WHERE status='processing' AND processing_until IS NOT NULL"
" AND processing_until < %s",
(now, now),
)
count = cur.rowcount
conn.commit()
return count
def finalize_ready(
self,
attachment_id: UUID,
pipeline_version: int,
variants: list[dict],
) -> None:
with connection() as conn:
with conn.cursor() as cur:
original = next(v for v in variants if v["name"] == "original")
cur.execute(
"UPDATE attachments SET status='ready', error_code=NULL,"
" object_key=%s, processing_until=NULL, updated_at=%s"
" WHERE id=%s",
(original["object_key"], now_utc(), attachment_id),
)
for v in variants:
if v["name"] == "original":
continue
cur.execute(
"INSERT INTO attachment_variants"
" (attachment_id, pipeline_version, variant, object_key,"
" content_type, size_bytes, width, height, sha256)"
" VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
" ON CONFLICT (attachment_id, pipeline_version, variant)"
" DO NOTHING",
(
attachment_id, pipeline_version, v["name"],
v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"],
),
)
conn.commit()
def finalize_failed(self, attachment_id: UUID, error_code: str) -> None:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE attachments SET status='failed', error_code=%s,"
" processing_until=NULL, updated_at=%s WHERE id=%s",
(error_code, now_utc(), attachment_id),
)
conn.commit()
def get_attachment(self, attachment_id: UUID) -> Optional[Attachment]:
with connection() as conn, conn.cursor() as cur:
cur.execute(
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes,"
" sha256, status, error_code, created_at, updated_at"
" FROM attachments WHERE id = %s",
(attachment_id,),
)
row = cur.fetchone()
if row is None:
return None
cur.execute(
"SELECT object_key FROM attachments WHERE id = %s", (attachment_id,),
)
obj_row = cur.fetchone()
att = _row_to_attachment(row)
att_variants: List[Variant] = [] # lab4: вариантов нет
att.variants = att_variants
# object_key нужен вызывающему для чтения из S3; отдаём через
# отдельный метод get_attachment_with_key.
return att
pair = self.get_attachment_with_key(attachment_id)
return pair[0] if pair else None
def get_attachment_with_key(
self, attachment_id: UUID
@@ -426,13 +470,12 @@ class PostgresStorage:
row = cur.fetchone()
if row is None:
return None
att = _row_to_attachment(row[:11])
att = _row_to_attachment(row[:11], self.get_attachment_variants(attachment_id))
return att, row[11]
def list_attachments_by_ids(
self, ids: List[UUID], chat_id: UUID
) -> dict[UUID, tuple[Attachment, str]]:
"""Достаём вложения этого чата. Возвращает {id: (Attachment, object_key)}."""
if not ids:
return {}
with connection() as conn, conn.cursor() as cur:
@@ -445,7 +488,6 @@ class PostgresStorage:
result: dict[UUID, tuple[Attachment, str]] = {}
for row in cur.fetchall():
att = _row_to_attachment(row[:11])
att.variants = []
result[row[0]] = (att, row[11])
return result
@@ -454,5 +496,69 @@ class PostgresStorage:
cur.execute("DELETE FROM attachments WHERE id = %s", (attachment_id,))
conn.commit()
def get_attachment_variants(self, attachment_id: UUID) -> list[dict]:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT variant, content_type, size_bytes, width, height, sha256"
" FROM attachment_variants WHERE attachment_id=%s"
" ORDER BY variant",
(attachment_id,),
)
return [
{
"name": r[0], "content_type": r[1], "size_bytes": r[2],
"width": r[3], "height": r[4], "sha256": r[5],
}
for r in cur.fetchall()
]
def get_staging_key(self, attachment_id: UUID) -> Optional[str]:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT object_key FROM attachments WHERE id=%s",
(attachment_id,),
)
row = cur.fetchone()
return row[0] if row else None
# ---------- Outbox (lab5 level 4) ----------
def list_unpublished_outbox(self, limit: int = 100) -> list[tuple[int, dict]]:
"""Возвращает [(outbox_id, payload_dict), ...].
Всегда dict; если payload в БД пришёл строкой (например, из старой
версии), распаковываем через json.loads.
"""
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT id, payload FROM outbox WHERE published_at IS NULL"
" ORDER BY id LIMIT %s",
(limit,),
)
rows = cur.fetchall()
result: list[tuple[int, dict]] = []
for r in rows:
raw = r[1]
if isinstance(raw, dict):
payload = raw
else:
payload = _json.loads(raw)
if not isinstance(payload, dict):
# защита от мусора — пропустить
continue
result.append((int(r[0]), payload))
return result
def mark_outbox_published(self, outbox_id: int) -> None:
with connection() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE outbox SET published_at=now() WHERE id=%s",
(outbox_id,),
)
conn.commit()
storage = PostgresStorage() # единственный экземпляр на процесс