forked from mtusi2026/mtusi2026lab5
341 lines
11 KiB
Python
341 lines
11 KiB
Python
"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO."""
|
|
|
|
from collections.abc import AsyncIterator, Iterator
|
|
from contextlib import contextmanager, suppress
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from uuid import UUID, uuid4
|
|
|
|
from anyio import CancelScope
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.request_capacity import RequestCapacity
|
|
from app.domain.chats import ensure_chat_member
|
|
from app.domain.attachment_jobs import enqueue_attachment_job
|
|
from app.domain.images import validate_filename, validate_image
|
|
from app.errors.exceptions import (
|
|
conflict,
|
|
invalid_request,
|
|
not_found,
|
|
payload_too_large,
|
|
rate_limited,
|
|
unavailable,
|
|
unsupported_media_type,
|
|
)
|
|
from app.errors.handlers import (
|
|
RECOVERABLE_STORAGE_ERRORS,
|
|
log_recoverable_error,
|
|
operation_timeout,
|
|
)
|
|
from app.schemas.attachment import Attachment, VARIANT_FILENAMES
|
|
from app.schemas.user import User
|
|
from app.storage import attachments as repository
|
|
from app.storage.s3 import MinioStorage
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CleanupResult:
|
|
examined: int
|
|
candidates: int
|
|
deleted: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OriginalFile:
|
|
path: Path
|
|
size_bytes: int
|
|
sha256: str
|
|
|
|
|
|
def require_storage(storage: MinioStorage | None) -> MinioStorage:
|
|
if storage is None:
|
|
raise unavailable("Объектное хранилище не настроено")
|
|
return storage
|
|
|
|
|
|
def validate_declared_size(value: str | None, maximum: int) -> None:
|
|
if value is not None:
|
|
if not value.isascii() or not value.isdecimal() or len(value) > 20:
|
|
raise invalid_request("Некорректный Content-Length")
|
|
if int(value) > maximum:
|
|
raise payload_too_large()
|
|
|
|
|
|
@contextmanager
|
|
def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
|
|
if not capacity.try_acquire():
|
|
raise rate_limited()
|
|
try:
|
|
yield
|
|
finally:
|
|
capacity.release()
|
|
|
|
|
|
async def receive_attachment(
|
|
session: AsyncSession,
|
|
storage: MinioStorage | None,
|
|
user: User,
|
|
chat_id: UUID,
|
|
*,
|
|
capacity: RequestCapacity,
|
|
filename: str,
|
|
content_type: str,
|
|
declared_length: str | None,
|
|
chunks: AsyncIterator[bytes],
|
|
) -> Attachment:
|
|
storage = require_storage(storage)
|
|
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
|
|
timeout_error = invalid_request("Истекло время загрузки")
|
|
with upload_slot(capacity):
|
|
async with operation_timeout(
|
|
storage.settings.upload_timeout_seconds, timeout_error
|
|
):
|
|
result = await upload_attachment(
|
|
session,
|
|
storage,
|
|
user,
|
|
chat_id,
|
|
filename=filename,
|
|
content_type=content_type,
|
|
chunks=chunks,
|
|
)
|
|
return result
|
|
|
|
|
|
async def read_original(
|
|
chunks: AsyncIterator[bytes],
|
|
path: Path,
|
|
maximum: int,
|
|
chunk_bytes: int,
|
|
) -> OriginalFile:
|
|
checksum = hashlib.sha256()
|
|
size = 0
|
|
with path.open("wb") as target:
|
|
async for chunk in chunks:
|
|
size += len(chunk)
|
|
if size > maximum:
|
|
raise payload_too_large()
|
|
checksum.update(chunk)
|
|
view = memoryview(chunk)
|
|
for offset in range(0, len(view), chunk_bytes):
|
|
target.write(view[offset : offset + chunk_bytes])
|
|
if size == 0:
|
|
raise unsupported_media_type("Пустой файл")
|
|
return OriginalFile(path, size, checksum.hexdigest())
|
|
|
|
|
|
async def compensate_upload(
|
|
session: AsyncSession, storage: MinioStorage, id: UUID
|
|
) -> None:
|
|
"""Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется."""
|
|
await session.rollback()
|
|
intent = await repository.lock_upload(session, id)
|
|
if intent is not None:
|
|
if await repository.get_by_key(session, intent.object_key) is None:
|
|
await storage.delete_object(intent.object_key)
|
|
await repository.finish_upload(session, id)
|
|
await session.commit()
|
|
|
|
|
|
async def recover_upload(
|
|
session: AsyncSession, storage: MinioStorage, id: UUID
|
|
) -> None:
|
|
"""Локальное восстановление после ошибки; журнал остаётся при отказе компенсации."""
|
|
try:
|
|
await compensate_upload(session, storage, id)
|
|
except RECOVERABLE_STORAGE_ERRORS as error:
|
|
log_recoverable_error("Upload compensation", error)
|
|
# Ошибка повторного rollback не должна скрыть исходную ошибку или отмену.
|
|
with suppress(*RECOVERABLE_STORAGE_ERRORS):
|
|
await session.rollback()
|
|
|
|
|
|
async def stage_attachment(
|
|
session: AsyncSession,
|
|
storage: MinioStorage,
|
|
user: User,
|
|
chat_id: UUID,
|
|
*,
|
|
original: OriginalFile,
|
|
filename: str,
|
|
content_type: str,
|
|
) -> Attachment:
|
|
"""Сохранить исходник в staging и атомарно принять metadata/job."""
|
|
id = uuid4()
|
|
key = f"staging/{uuid4().hex}"
|
|
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
|
|
await repository.reserve_upload(session, id, key)
|
|
await session.commit()
|
|
try:
|
|
if await repository.lock_upload(session, id) is None:
|
|
raise unavailable("Загрузка отменена до записи оригинала")
|
|
await storage.put_file(
|
|
key,
|
|
original.path,
|
|
size_bytes=original.size_bytes,
|
|
content_type=content_type,
|
|
sha256=original.sha256,
|
|
)
|
|
record = await repository.create_attachment(
|
|
session,
|
|
id=id,
|
|
chat_id=chat_id,
|
|
owner_id=user.id,
|
|
object_key=f"originals/{id.hex}/v1",
|
|
status="queued",
|
|
filename=filename,
|
|
content_type=content_type,
|
|
size_bytes=original.size_bytes,
|
|
sha256=original.sha256,
|
|
)
|
|
await enqueue_attachment_job(session, id, key)
|
|
await repository.finish_upload(session, id)
|
|
# Снимок queued строится до COMMIT, даже если worker очень быстрый.
|
|
result = Attachment.model_validate(record)
|
|
await session.commit()
|
|
except BaseException:
|
|
# Отмена тоже требует дождаться SDK и компенсировать незавершённый PUT.
|
|
with CancelScope(shield=True):
|
|
await recover_upload(session, storage, id)
|
|
raise
|
|
return result
|
|
|
|
|
|
async def upload_attachment(
|
|
session: AsyncSession,
|
|
storage: MinioStorage,
|
|
user: User,
|
|
chat_id: UUID,
|
|
*,
|
|
filename: str,
|
|
content_type: str,
|
|
chunks: AsyncIterator[bytes],
|
|
) -> Attachment:
|
|
await ensure_chat_member(session, user, chat_id)
|
|
validate_filename(filename)
|
|
if content_type not in {"image/png", "image/jpeg"}:
|
|
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
|
|
# Не держим соединение PostgreSQL во время чтения медленного клиента.
|
|
await session.rollback()
|
|
settings = storage.settings
|
|
with TemporaryDirectory(prefix="messenger-upload-") as directory:
|
|
original = await read_original(
|
|
chunks,
|
|
Path(directory) / "original",
|
|
settings.max_upload_bytes,
|
|
settings.chunk_bytes,
|
|
)
|
|
await validate_image(original.path, content_type, settings)
|
|
result = await stage_attachment(
|
|
session,
|
|
storage,
|
|
user,
|
|
chat_id,
|
|
original=original,
|
|
filename=filename,
|
|
content_type=content_type,
|
|
)
|
|
return result
|
|
|
|
|
|
async def get_attachment(
|
|
session: AsyncSession, user: User, attachment_id: UUID
|
|
) -> Attachment:
|
|
record = await repository.get_attachment(session, attachment_id)
|
|
if record is None:
|
|
raise not_found()
|
|
await ensure_chat_member(session, user, record.chat_id)
|
|
return Attachment.model_validate(record)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DownloadInfo:
|
|
filename: str
|
|
content_type: str
|
|
size_bytes: int
|
|
|
|
|
|
async def download_attachment(
|
|
session: AsyncSession,
|
|
storage: MinioStorage | None,
|
|
user: User,
|
|
attachment_id: UUID,
|
|
*,
|
|
variant: str = "original",
|
|
) -> tuple[DownloadInfo, AsyncIterator[bytes]]:
|
|
result = await get_attachment(session, user, attachment_id)
|
|
if result.status != "ready":
|
|
raise conflict("Вложение пока недоступно для скачивания")
|
|
storage = require_storage(storage)
|
|
record = await repository.get_attachment(session, attachment_id)
|
|
if record is None:
|
|
raise not_found()
|
|
if variant == "original":
|
|
key = record.object_key
|
|
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
|
|
else:
|
|
selected = next(
|
|
(item for item in result.variants if item.name == variant), None
|
|
)
|
|
keys = {
|
|
"thumbnail": record.thumbnail_key,
|
|
"crop": record.crop_key,
|
|
"optimized": record.optimized_key,
|
|
}
|
|
key = keys.get(variant)
|
|
if selected is None or key is None:
|
|
raise not_found()
|
|
info = DownloadInfo(
|
|
VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes
|
|
)
|
|
await session.rollback()
|
|
stored = await storage.stat_object(key)
|
|
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
|
|
raise unavailable("Файл временно недоступен")
|
|
stream = await storage.open_object(key)
|
|
return info, stream
|
|
|
|
|
|
async def validate_message_attachments(
|
|
session: AsyncSession,
|
|
user: User,
|
|
chat_id: UUID,
|
|
attachment_ids: list[UUID],
|
|
) -> None:
|
|
await ensure_chat_member(session, user, chat_id)
|
|
records = await repository.get_for_chat(session, chat_id, attachment_ids)
|
|
if len(records) != len(attachment_ids):
|
|
raise not_found()
|
|
if any(record.status == "failed" for record in records):
|
|
raise conflict("Обработка вложения завершилась ошибкой")
|
|
|
|
|
|
async def reconcile_attachments(
|
|
session: AsyncSession,
|
|
storage: MinioStorage,
|
|
*,
|
|
batch_size: int = 20,
|
|
dry_run: bool = True,
|
|
) -> CleanupResult:
|
|
if not 1 <= batch_size <= 100:
|
|
raise invalid_request("batch_size должен быть от 1 до 100")
|
|
intents = await repository.stale_uploads(
|
|
session, storage.settings.cleanup_grace_seconds, batch_size
|
|
)
|
|
candidates = deleted = 0
|
|
for intent in intents:
|
|
if await repository.get_by_key(session, intent.object_key) is None:
|
|
candidates += 1
|
|
if not dry_run:
|
|
await storage.delete_object(intent.object_key)
|
|
deleted += 1
|
|
if not dry_run:
|
|
await repository.finish_upload(session, intent.id)
|
|
if dry_run:
|
|
await session.rollback()
|
|
else:
|
|
await session.commit()
|
|
return CleanupResult(len(intents), candidates, deleted)
|