diff --git a/app/core/cleanup.py b/app/core/cleanup.py index b0faf884..7e755075 100644 --- a/app/core/cleanup.py +++ b/app/core/cleanup.py @@ -20,13 +20,13 @@ async def cleanup_once(factory, storage, apply: bool) -> bool: result = await reconcile_attachments(session, storage, dry_run=not apply) print(json.dumps(asdict(result)), flush=True) except RECOVERABLE_STORAGE_ERRORS as error: - log_recoverable_error('Reconciliation', error) + log_recoverable_error("Reconciliation", error) completed = False return completed async def run(once: bool, apply: bool) -> None: - settings = Settings() + settings = Settings() # type: ignore s3_settings = S3Settings() engine = create_db_engine(settings) factory = create_session_factory(engine) @@ -45,12 +45,16 @@ async def run(once: bool, apply: bool) -> None: def main() -> None: - parser = argparse.ArgumentParser(description='Сверка журнала загрузок MinIO/БД') - parser.add_argument('--once', action='store_true') - parser.add_argument('--apply', action='store_true', help='удалять безопасные кандидаты; по умолчанию dry-run') + parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД") + parser.add_argument("--once", action="store_true") + parser.add_argument( + "--apply", + action="store_true", + help="удалять безопасные кандидаты; по умолчанию dry-run", + ) args = parser.parse_args() asyncio.run(run(args.once, args.apply)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/app/core/config.py b/app/core/config.py index 9cc8dae2..6b743709 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -3,13 +3,24 @@ from pathlib import Path from typing import Literal, Self from pydantic import Field, SecretStr, model_validator -from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) from sqlalchemy import URL from app.core.s3 import S3Settings ASYNC_DRIVER = "postgresql+asyncpg" SECRETS_DIR = next( - (path for path in (Path("/run/secrets"), Path(__file__).resolve().parents[2] / "secrets") if (path / "db_password").is_file()), + ( + path + for path in ( + Path("/run/secrets"), + Path(__file__).resolve().parents[2] / "secrets", + ) + if (path / "db_password").is_file() + ), None, ) @@ -88,7 +99,9 @@ class Settings(BaseSettings): @model_validator(mode="after") def _require_database_source(self) -> Self: if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8: - raise ValueError("Drain и graceful shutdown должны суммарно занимать менее 8 секунд") + raise ValueError( + "Drain и graceful shutdown должны суммарно занимать менее 8 секунд" + ) if self.database_url is not None: if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER): raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://") diff --git a/app/core/database.py b/app/core/database.py index 294a165a..c7a65381 100644 --- a/app/core/database.py +++ b/app/core/database.py @@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import ( from app.core.config import Settings + def create_db_engine(settings: Settings) -> AsyncEngine: return create_async_engine( settings.get_database_url(), @@ -25,5 +26,3 @@ def create_db_engine(settings: Settings) -> AsyncEngine: def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False) - - diff --git a/app/core/image_worker.py b/app/core/image_worker.py index 2d043bf2..a74795c3 100644 --- a/app/core/image_worker.py +++ b/app/core/image_worker.py @@ -1,4 +1,4 @@ -"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU.""" +"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU""" from pathlib import Path import resource diff --git a/app/domain/attachments.py b/app/domain/attachments.py index d09909d8..3216c636 100644 --- a/app/domain/attachments.py +++ b/app/domain/attachments.py @@ -23,12 +23,17 @@ from app.errors.exceptions import ( unavailable, unsupported_media_type, ) -from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error, operation_timeout +from app.errors.handlers import ( + RECOVERABLE_STORAGE_ERRORS, + log_recoverable_error, + operation_timeout, +) from app.schemas.attachment import Attachment from app.schemas.user import User from app.storage import attachments as repository from app.storage.s3 import MinioStorage + @dataclass(frozen=True) class CleanupResult: examined: int @@ -45,14 +50,14 @@ class OriginalFile: def require_storage(storage: MinioStorage | None) -> MinioStorage: if storage is None: - raise unavailable('Объектное хранилище не настроено') + raise unavailable("Объектное хранилище не настроено") return storage def validate_declared_size(value: str | None, maximum: int) -> None: if value is not None: if not value.isascii() or not value.isdecimal() or len(value) > 20: - raise invalid_request('Некорректный Content-Length') + raise invalid_request("Некорректный Content-Length") if int(value) > maximum: raise payload_too_large() @@ -68,28 +73,45 @@ def upload_slot(capacity: RequestCapacity) -> Iterator[None]: async def receive_attachment( - session: AsyncSession, storage: MinioStorage | None, user: User, chat_id: UUID, - *, capacity: RequestCapacity, filename: str, content_type: str, - declared_length: str | None, chunks: AsyncIterator[bytes], + session: AsyncSession, + storage: MinioStorage | None, + user: User, + chat_id: UUID, + *, + capacity: RequestCapacity, + filename: str, + content_type: str, + declared_length: str | None, + chunks: AsyncIterator[bytes], ) -> Attachment: storage = require_storage(storage) validate_declared_size(declared_length, storage.settings.max_upload_bytes) - timeout_error = invalid_request('Истекло время загрузки') + timeout_error = invalid_request("Истекло время загрузки") with upload_slot(capacity): - async with operation_timeout(storage.settings.upload_timeout_seconds, timeout_error): + async with operation_timeout( + storage.settings.upload_timeout_seconds, timeout_error + ): result = await upload_attachment( - session, storage, user, chat_id, filename=filename, - content_type=content_type, chunks=chunks, + session, + storage, + user, + chat_id, + filename=filename, + content_type=content_type, + chunks=chunks, ) return result async def read_original( - chunks: AsyncIterator[bytes], path: Path, maximum: int, chunk_bytes: int, + chunks: AsyncIterator[bytes], + path: Path, + maximum: int, + chunk_bytes: int, ) -> OriginalFile: checksum = hashlib.sha256() size = 0 - with path.open('wb') as target: + with path.open("wb") as target: async for chunk in chunks: size += len(chunk) if size > maximum: @@ -97,13 +119,15 @@ async def read_original( checksum.update(chunk) view = memoryview(chunk) for offset in range(0, len(view), chunk_bytes): - target.write(view[offset:offset + chunk_bytes]) + target.write(view[offset : offset + chunk_bytes]) if size == 0: - raise unsupported_media_type('Пустой файл') + raise unsupported_media_type("Пустой файл") return OriginalFile(path, size, checksum.hexdigest()) -async def compensate_upload(session: AsyncSession, storage: MinioStorage, id: UUID) -> None: +async def compensate_upload( + session: AsyncSession, storage: MinioStorage, id: UUID +) -> None: """Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется.""" await session.rollback() intent = await repository.lock_upload(session, id) @@ -114,37 +138,54 @@ async def compensate_upload(session: AsyncSession, storage: MinioStorage, id: UU await session.commit() -async def recover_upload(session: AsyncSession, storage: MinioStorage, id: UUID) -> None: +async def recover_upload( + session: AsyncSession, storage: MinioStorage, id: UUID +) -> None: """Локальное восстановление после ошибки; журнал остаётся при отказе компенсации.""" try: await compensate_upload(session, storage, id) except RECOVERABLE_STORAGE_ERRORS as error: - log_recoverable_error('Upload compensation', error) + log_recoverable_error("Upload compensation", error) # Ошибка повторного rollback не должна скрыть исходную ошибку или отмену. with suppress(*RECOVERABLE_STORAGE_ERRORS): await session.rollback() async def publish_original( - session: AsyncSession, storage: MinioStorage, user: User, chat_id: UUID, - *, original: OriginalFile, filename: str, content_type: str, + session: AsyncSession, + storage: MinioStorage, + user: User, + chat_id: UUID, + *, + original: OriginalFile, + filename: str, + content_type: str, ) -> Attachment: id = uuid4() - key = f'originals/{uuid4().hex}' + key = f"originals/{uuid4().hex}" # Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT. await repository.reserve_upload(session, id, key) await session.commit() try: if await repository.lock_upload(session, id) is None: - raise unavailable('Загрузка отменена до записи оригинала') + raise unavailable("Загрузка отменена до записи оригинала") await storage.put_file( - key, original.path, size_bytes=original.size_bytes, - content_type=content_type, sha256=original.sha256, + key, + original.path, + size_bytes=original.size_bytes, + content_type=content_type, + sha256=original.sha256, ) record = await repository.create_attachment( - session, id=id, chat_id=chat_id, owner_id=user.id, object_key=key, - filename=filename, content_type=content_type, - size_bytes=original.size_bytes, sha256=original.sha256, + session, + id=id, + chat_id=chat_id, + owner_id=user.id, + object_key=key, + filename=filename, + content_type=content_type, + size_bytes=original.size_bytes, + sha256=original.sha256, ) await repository.finish_upload(session, id) result = Attachment.model_validate(record) @@ -158,29 +199,45 @@ async def publish_original( async def upload_attachment( - session: AsyncSession, storage: MinioStorage, user: User, chat_id: UUID, - *, filename: str, content_type: str, chunks: AsyncIterator[bytes], + session: AsyncSession, + storage: MinioStorage, + user: User, + chat_id: UUID, + *, + filename: str, + content_type: str, + chunks: AsyncIterator[bytes], ) -> Attachment: await ensure_chat_member(session, user, chat_id) validate_filename(filename) - if content_type not in {'image/png', 'image/jpeg'}: - raise unsupported_media_type('Принимаются только image/png и image/jpeg') + if content_type not in {"image/png", "image/jpeg"}: + raise unsupported_media_type("Принимаются только image/png и image/jpeg") # Не держим соединение PostgreSQL во время чтения медленного клиента. await session.rollback() settings = storage.settings - with TemporaryDirectory(prefix='messenger-upload-') as directory: + with TemporaryDirectory(prefix="messenger-upload-") as directory: original = await read_original( - chunks, Path(directory) / 'original', settings.max_upload_bytes, settings.chunk_bytes, + chunks, + Path(directory) / "original", + settings.max_upload_bytes, + settings.chunk_bytes, ) await validate_image(original.path, content_type, settings) result = await publish_original( - session, storage, user, chat_id, original=original, - filename=filename, content_type=content_type, + session, + storage, + user, + chat_id, + original=original, + filename=filename, + content_type=content_type, ) return result -async def get_attachment(session: AsyncSession, user: User, attachment_id: UUID) -> Attachment: +async def get_attachment( + session: AsyncSession, user: User, attachment_id: UUID +) -> Attachment: record = await repository.get_attachment(session, attachment_id) if record is None: raise not_found() @@ -189,41 +246,59 @@ async def get_attachment(session: AsyncSession, user: User, attachment_id: UUID) async def download_attachment( - session: AsyncSession, storage: MinioStorage | None, user: User, attachment_id: UUID, - *, variant: str = 'original', + session: AsyncSession, + storage: MinioStorage | None, + user: User, + attachment_id: UUID, + *, + variant: str = "original", ) -> tuple[Attachment, AsyncIterator[bytes]]: result = await get_attachment(session, user, attachment_id) - if variant != 'original': + if variant != "original": raise not_found() storage = require_storage(storage) record = await repository.get_attachment(session, attachment_id) + if record is None: + raise not_found() key = record.object_key await session.rollback() info = await storage.stat_object(key) - if info is None or info.size_bytes != result.size_bytes or info.content_type != result.content_type: - raise unavailable('Оригинал временно недоступен') + if ( + info is None + or info.size_bytes != result.size_bytes + or info.content_type != result.content_type + ): + raise unavailable("Оригинал временно недоступен") stream = await storage.open_object(key) return result, stream async def validate_message_attachments( - session: AsyncSession, user: User, chat_id: UUID, attachment_ids: list[UUID], + session: AsyncSession, + user: User, + chat_id: UUID, + attachment_ids: list[UUID], ) -> None: await ensure_chat_member(session, user, chat_id) records = await repository.get_for_chat(session, chat_id, attachment_ids) if len(records) != len(attachment_ids): raise not_found() - if any(record.status != 'ready' for record in records): - raise conflict('Вложение ещё не готово') + if any(record.status != "ready" for record in records): + raise conflict("Вложение ещё не готово") async def reconcile_attachments( - session: AsyncSession, storage: MinioStorage, *, batch_size: int = 20, + session: AsyncSession, + storage: MinioStorage, + *, + batch_size: int = 20, dry_run: bool = True, ) -> CleanupResult: if not 1 <= batch_size <= 100: - raise invalid_request('batch_size должен быть от 1 до 100') - intents = await repository.stale_uploads(session, storage.settings.cleanup_grace_seconds, batch_size) + raise invalid_request("batch_size должен быть от 1 до 100") + intents = await repository.stale_uploads( + session, storage.settings.cleanup_grace_seconds, batch_size + ) candidates = deleted = 0 for intent in intents: if await repository.get_by_key(session, intent.object_key) is None: diff --git a/app/domain/image_validation.py b/app/domain/image_validation.py index 58000391..c004d295 100644 --- a/app/domain/image_validation.py +++ b/app/domain/image_validation.py @@ -10,13 +10,14 @@ from app.errors.exceptions import payload_too_large, unsupported_media_type def decode_image(path: Path, content_type: str, max_pixels: int) -> None: Image.MAX_IMAGE_PIXELS = max_pixels - warnings.simplefilter('error', Image.DecompressionBombWarning) - with Image.open(path, formats=['PNG', 'JPEG']) as image: + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(path, formats=["PNG", "JPEG"]) as image: if image.width * image.height > max_pixels: - raise payload_too_large('Превышен лимит пикселей изображения') - if Image.MIME.get(image.format) != content_type: - raise unsupported_media_type('Содержимое не соответствует Content-Type') + raise payload_too_large("Превышен лимит пикселей изображения") + image_format = image.format + if image_format is None or Image.MIME.get(image_format) != content_type: + raise unsupported_media_type("Содержимое не соответствует Content-Type") image.verify() # Для JPEG одной verify() недостаточно: декодировать весь оригинал. - with Image.open(path, formats=['PNG', 'JPEG']) as image: + with Image.open(path, formats=["PNG", "JPEG"]) as image: image.load() diff --git a/app/domain/images.py b/app/domain/images.py index 0cc8bc3b..4436656f 100644 --- a/app/domain/images.py +++ b/app/domain/images.py @@ -8,24 +8,34 @@ import unicodedata from anyio import CancelScope from app.core.s3 import S3Settings -from app.errors.exceptions import image_validation_error, invalid_request, payload_too_large +from app.errors.exceptions import ( + image_validation_error, + invalid_request, + payload_too_large, +) from app.errors.handlers import operation_timeout def validate_filename(filename: str) -> None: if not 1 <= len(filename) <= 255 or any( - char in '/\\' or unicodedata.category(char) in {'Cc', 'Cf', 'Cs'} for char in filename + char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"} + for char in filename ): - raise invalid_request('Недопустимое имя файла') + raise invalid_request("Недопустимое имя файла") async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None: process = await asyncio.create_subprocess_exec( - sys.executable, '-m', 'app.core.image_worker', str(path), content_type, - str(settings.max_image_pixels), stdout=asyncio.subprocess.DEVNULL, + sys.executable, + "-m", + "app.core.image_worker", + str(path), + content_type, + str(settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, ) - error = payload_too_large('Превышен бюджет проверки изображения') + error = payload_too_large("Превышен бюджет проверки изображения") try: async with operation_timeout(settings.image_timeout_seconds, error): code = await process.wait() diff --git a/app/models/attachment.py b/app/models/attachment.py index df9456e3..3b7c678d 100644 --- a/app/models/attachment.py +++ b/app/models/attachment.py @@ -13,7 +13,9 @@ class Attachment(Base): __tablename__ = "attachments" __table_args__ = ( CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"), - CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name="content_type"), + CheckConstraint( + "content_type IN ('image/png', 'image/jpeg')", name="content_type" + ), CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"), CheckConstraint("status = 'ready'", name="status"), Index("ix_attachments_chat_id", "chat_id"), @@ -35,8 +37,12 @@ class Attachment(Base): class MessageAttachment(Base): __tablename__ = "message_attachments" - message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True) - attachment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("attachments.id"), primary_key=True) + message_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True + ) + attachment_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("attachments.id"), primary_key=True + ) position: Mapped[int] diff --git a/app/storage/attachments.py b/app/storage/attachments.py index 95cd219c..3ef1765f 100644 --- a/app/storage/attachments.py +++ b/app/storage/attachments.py @@ -16,7 +16,9 @@ async def create_attachment(session: AsyncSession, **values) -> Attachment: return record -async def get_attachment(session: AsyncSession, attachment_id: UUID) -> Attachment | None: +async def get_attachment( + session: AsyncSession, attachment_id: UUID +) -> Attachment | None: return await session.get(Attachment, attachment_id) @@ -24,27 +26,43 @@ async def get_by_key(session: AsyncSession, key: str) -> Attachment | None: return await session.scalar(select(Attachment).where(Attachment.object_key == key)) -async def get_for_chat(session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]) -> list[Attachment]: +async def get_for_chat( + session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID] +) -> list[Attachment]: records = [] if attachment_ids: - records = list(await session.scalars(select(Attachment).where( - Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id, - ))) + records = list( + await session.scalars( + select(Attachment).where( + Attachment.id.in_(attachment_ids), + Attachment.chat_id == chat_id, + ) + ) + ) return records -async def bind_attachments(session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]) -> None: - session.add_all(MessageAttachment(message_id=message_id, attachment_id=id, position=position) - for position, id in enumerate(attachment_ids)) +async def bind_attachments( + session: AsyncSession, message_id: UUID, attachment_ids: list[UUID] +) -> None: + session.add_all( + MessageAttachment(message_id=message_id, attachment_id=id, position=position) + for position, id in enumerate(attachment_ids) + ) await session.flush() -async def message_attachment_ids(session: AsyncSession, message_ids: list[UUID]) -> dict[UUID, list[UUID]]: +async def message_attachment_ids( + session: AsyncSession, message_ids: list[UUID] +) -> dict[UUID, list[UUID]]: result: dict[UUID, list[UUID]] = {id: [] for id in message_ids} if not message_ids: return result - rows = await session.execute(select(MessageAttachment).where( - MessageAttachment.message_id.in_(message_ids)).order_by(MessageAttachment.position)) + rows = await session.execute( + select(MessageAttachment) + .where(MessageAttachment.message_id.in_(message_ids)) + .order_by(MessageAttachment.position) + ) for link in rows.scalars(): result[link.message_id].append(link.attachment_id) return result @@ -56,15 +74,26 @@ async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None: async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None: - return await session.scalar(select(UploadIntent).where(UploadIntent.id == id).with_for_update()) + return await session.scalar( + select(UploadIntent).where(UploadIntent.id == id).with_for_update() + ) async def finish_upload(session: AsyncSession, id: UUID) -> None: await session.execute(delete(UploadIntent).where(UploadIntent.id == id)) -async def stale_uploads(session: AsyncSession, grace_seconds: int, limit: int) -> list[UploadIntent]: - result = await session.scalars(select(UploadIntent).where( - UploadIntent.created_at < func.clock_timestamp() - timedelta(seconds=grace_seconds), - ).order_by(UploadIntent.created_at).limit(limit).with_for_update(skip_locked=True)) +async def stale_uploads( + session: AsyncSession, grace_seconds: int, limit: int +) -> list[UploadIntent]: + result = await session.scalars( + select(UploadIntent) + .where( + UploadIntent.created_at + < func.clock_timestamp() - timedelta(seconds=grace_seconds), + ) + .order_by(UploadIntent.created_at) + .limit(limit) + .with_for_update(skip_locked=True) + ) return list(result) diff --git a/app/storage/s3.py b/app/storage/s3.py index 42ce6258..b3355a31 100644 --- a/app/storage/s3.py +++ b/app/storage/s3.py @@ -19,7 +19,9 @@ from app.core.s3 import S3Settings T = TypeVar("T") -async def run_sync(operation: Callable[..., T], *args, discard_result=None, **kwargs) -> T: +async def run_sync( + operation: Callable[..., T], *args, discard_result=None, **kwargs +) -> T: """При отмене дождаться потока: иначе PUT может догнать компенсацию удаления.""" task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs)) try: @@ -31,7 +33,9 @@ async def run_sync(operation: Callable[..., T], *args, discard_result=None, **kw return result -async def settle_cancelled_call(task: asyncio.Task, discard_result: Callable | None) -> None: +async def settle_cancelled_call( + task: asyncio.Task, discard_result: Callable | None +) -> None: """wait не переносит исключение SDK; повторная отмена не прерывает компенсацию.""" while not task.done(): with suppress(asyncio.CancelledError): @@ -43,10 +47,12 @@ async def settle_cancelled_call(task: asyncio.Task, discard_result: Callable | N @dataclass(frozen=True) class ObjectInfo: + """Поля S3 сохраняют необязательность SDK; согласованность проверяет domain.""" + key: str - size_bytes: int - content_type: str - last_modified: datetime + size_bytes: int | None + content_type: str | None + last_modified: datetime | None def close_response(response) -> None: @@ -74,7 +80,9 @@ class ObjectStream: if remaining <= 0: raise TimeoutError async with asyncio.timeout(remaining): - chunk = await self.storage._call(lambda: self.response.read(self.storage.settings.chunk_bytes)) + chunk = await self.storage._call( + lambda: self.response.read(self.storage.settings.chunk_bytes) + ) if not chunk: await self.aclose() raise StopAsyncIteration @@ -95,35 +103,52 @@ class MinioStorage: # Клиент создаётся в event loop один раз, не конкурентно в worker threads. url = urlsplit(settings.endpoint) self._http = urllib3.PoolManager( - timeout=urllib3.Timeout(connect=settings.connect_timeout_seconds, read=settings.read_timeout_seconds), - retries=urllib3.Retry(total=0), maxsize=32, + timeout=urllib3.Timeout( + connect=settings.connect_timeout_seconds, + read=settings.read_timeout_seconds, + ), + retries=urllib3.Retry(total=0), + maxsize=32, ) self._health_http = urllib3.PoolManager( - timeout=urllib3.Timeout(connect=0.25, read=0.5), retries=urllib3.Retry(total=0), + timeout=urllib3.Timeout(connect=0.25, read=0.5), + retries=urllib3.Retry(total=0), ) self._client = Minio( - url.netloc, secure=url.scheme == "https", region=settings.region, + url.netloc, + secure=url.scheme == "https", + region=settings.region, access_key=settings.access_key_id_file.read_text().strip(), - secret_key=settings.secret_access_key_file.read_text().strip(), http_client=self._http, + secret_key=settings.secret_access_key_file.read_text().strip(), + http_client=self._http, ) async def _call(self, operation: Callable, *, discard_result=None): return await run_sync(operation, discard_result=discard_result) - async def put_file(self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str) -> None: + async def put_file( + self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str + ) -> None: def put(): with path.open("rb") as source: # Один PUT: размер части равен максимальному разрешённому файлу. self._client.put_object( - self.settings.bucket, key, source, size_bytes, - content_type=content_type, metadata={"sha256": sha256}, - part_size=10_485_760, num_parallel_uploads=1, + self.settings.bucket, + key, + source, + size_bytes, + content_type=content_type, + metadata={"sha256": sha256}, + part_size=10_485_760, + num_parallel_uploads=1, ) + await self._call(put) async def open_object(self, key: str) -> ObjectStream: response = await self._call( - lambda: self._client.get_object(self.settings.bucket, key), discard_result=close_response, + lambda: self._client.get_object(self.settings.bucket, key), + discard_result=close_response, ) return ObjectStream(self, response) @@ -131,15 +156,19 @@ class MinioStorage: def stat(): info = self._client.stat_object(self.settings.bucket, key) return ObjectInfo(key, info.size, info.content_type, info.last_modified) + return await self._call(stat) async def delete_object(self, key: str) -> None: await self._call(lambda: self._client.remove_object(self.settings.bucket, key)) async def check_available(self) -> bool: - status = await self._call(lambda: self._health_http.request( - "GET", self.settings.endpoint.rstrip("/") + "/minio/health/ready", - ).status) + status = await self._call( + lambda: self._health_http.request( + "GET", + self.settings.endpoint.rstrip("/") + "/minio/health/ready", + ).status + ) return status == 200 async def aclose(self) -> None: