Files
mtusi2026lab5-fork/app/domain/attachments.py
T
2026-11-07 16:10:32 +03:00

316 lines
10 KiB
Python

"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO."""
from collections.abc import AsyncIterator, Iterator
from contextlib import contextmanager, suppress
from dataclasses import dataclass
import hashlib
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
from anyio import CancelScope
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.request_capacity import RequestCapacity
from app.domain.chats import ensure_chat_member
from app.domain.images import validate_filename, validate_image
from app.errors.exceptions import (
conflict,
invalid_request,
not_found,
payload_too_large,
rate_limited,
unavailable,
unsupported_media_type,
)
from app.errors.handlers import (
RECOVERABLE_STORAGE_ERRORS,
log_recoverable_error,
operation_timeout,
)
from app.schemas.attachment import Attachment
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import MinioStorage
@dataclass(frozen=True)
class CleanupResult:
examined: int
candidates: int
deleted: int
@dataclass(frozen=True)
class OriginalFile:
path: Path
size_bytes: int
sha256: str
def require_storage(storage: MinioStorage | None) -> MinioStorage:
if storage is None:
raise unavailable("Объектное хранилище не настроено")
return storage
def validate_declared_size(value: str | None, maximum: int) -> None:
if value is not None:
if not value.isascii() or not value.isdecimal() or len(value) > 20:
raise invalid_request("Некорректный Content-Length")
if int(value) > maximum:
raise payload_too_large()
@contextmanager
def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
if not capacity.try_acquire():
raise rate_limited()
try:
yield
finally:
capacity.release()
async def receive_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
chat_id: UUID,
*,
capacity: RequestCapacity,
filename: str,
content_type: str,
declared_length: str | None,
chunks: AsyncIterator[bytes],
) -> Attachment:
storage = require_storage(storage)
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
timeout_error = invalid_request("Истекло время загрузки")
with upload_slot(capacity):
async with operation_timeout(
storage.settings.upload_timeout_seconds, timeout_error
):
result = await upload_attachment(
session,
storage,
user,
chat_id,
filename=filename,
content_type=content_type,
chunks=chunks,
)
return result
async def read_original(
chunks: AsyncIterator[bytes],
path: Path,
maximum: int,
chunk_bytes: int,
) -> OriginalFile:
checksum = hashlib.sha256()
size = 0
with path.open("wb") as target:
async for chunk in chunks:
size += len(chunk)
if size > maximum:
raise payload_too_large()
checksum.update(chunk)
view = memoryview(chunk)
for offset in range(0, len(view), chunk_bytes):
target.write(view[offset : offset + chunk_bytes])
if size == 0:
raise unsupported_media_type("Пустой файл")
return OriginalFile(path, size, checksum.hexdigest())
async def compensate_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется."""
await session.rollback()
intent = await repository.lock_upload(session, id)
if intent is not None:
if await repository.get_by_key(session, intent.object_key) is None:
await storage.delete_object(intent.object_key)
await repository.finish_upload(session, id)
await session.commit()
async def recover_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Локальное восстановление после ошибки; журнал остаётся при отказе компенсации."""
try:
await compensate_upload(session, storage, id)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Upload compensation", error)
# Ошибка повторного rollback не должна скрыть исходную ошибку или отмену.
with suppress(*RECOVERABLE_STORAGE_ERRORS):
await session.rollback()
async def publish_original(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
original: OriginalFile,
filename: str,
content_type: str,
) -> Attachment:
id = uuid4()
key = f"originals/{uuid4().hex}"
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
await repository.reserve_upload(session, id, key)
await session.commit()
try:
if await repository.lock_upload(session, id) is None:
raise unavailable("Загрузка отменена до записи оригинала")
await storage.put_file(
key,
original.path,
size_bytes=original.size_bytes,
content_type=content_type,
sha256=original.sha256,
)
record = await repository.create_attachment(
session,
id=id,
chat_id=chat_id,
owner_id=user.id,
object_key=key,
filename=filename,
content_type=content_type,
size_bytes=original.size_bytes,
sha256=original.sha256,
)
await repository.finish_upload(session, id)
result = Attachment.model_validate(record)
await session.commit()
except BaseException:
# Отмена тоже требует дождаться SDK и компенсировать незавершённый PUT.
with CancelScope(shield=True):
await recover_upload(session, storage, id)
raise
return result
async def upload_attachment(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
filename: str,
content_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
await ensure_chat_member(session, user, chat_id)
validate_filename(filename)
if content_type not in {"image/png", "image/jpeg"}:
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
# Не держим соединение PostgreSQL во время чтения медленного клиента.
await session.rollback()
settings = storage.settings
with TemporaryDirectory(prefix="messenger-upload-") as directory:
original = await read_original(
chunks,
Path(directory) / "original",
settings.max_upload_bytes,
settings.chunk_bytes,
)
await validate_image(original.path, content_type, settings)
result = await publish_original(
session,
storage,
user,
chat_id,
original=original,
filename=filename,
content_type=content_type,
)
return result
async def get_attachment(
session: AsyncSession, user: User, attachment_id: UUID
) -> Attachment:
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
await ensure_chat_member(session, user, record.chat_id)
return Attachment.model_validate(record)
async def download_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
attachment_id: UUID,
*,
variant: str = "original",
) -> tuple[Attachment, AsyncIterator[bytes]]:
result = await get_attachment(session, user, attachment_id)
if variant != "original":
raise not_found()
storage = require_storage(storage)
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
key = record.object_key
await session.rollback()
info = await storage.stat_object(key)
if (
info is None
or info.size_bytes != result.size_bytes
or info.content_type != result.content_type
):
raise unavailable("Оригинал временно недоступен")
stream = await storage.open_object(key)
return result, stream
async def validate_message_attachments(
session: AsyncSession,
user: User,
chat_id: UUID,
attachment_ids: list[UUID],
) -> None:
await ensure_chat_member(session, user, chat_id)
records = await repository.get_for_chat(session, chat_id, attachment_ids)
if len(records) != len(attachment_ids):
raise not_found()
if any(record.status != "ready" for record in records):
raise conflict("Вложение ещё не готово")
async def reconcile_attachments(
session: AsyncSession,
storage: MinioStorage,
*,
batch_size: int = 20,
dry_run: bool = True,
) -> CleanupResult:
if not 1 <= batch_size <= 100:
raise invalid_request("batch_size должен быть от 1 до 100")
intents = await repository.stale_uploads(
session, storage.settings.cleanup_grace_seconds, batch_size
)
candidates = deleted = 0
for intent in intents:
if await repository.get_by_key(session, intent.object_key) is None:
candidates += 1
if not dry_run:
await storage.delete_object(intent.object_key)
deleted += 1
if not dry_run:
await repository.finish_upload(session, intent.id)
if dry_run:
await session.rollback()
else:
await session.commit()
return CleanupResult(len(intents), candidates, deleted)