"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции.""" import asyncio from collections.abc import Callable from contextlib import suppress from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import TypeVar import time from urllib.parse import urlsplit from anyio import CancelScope from minio import Minio import urllib3 from app.core.s3 import S3Settings T = TypeVar("T") async def run_sync( operation: Callable[..., T], *args, discard_result=None, **kwargs ) -> T: """При отмене дождаться потока: иначе PUT может догнать компенсацию удаления.""" task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs)) try: result = await asyncio.shield(task) except asyncio.CancelledError: with CancelScope(shield=True): await settle_cancelled_call(task, discard_result) raise return result async def settle_cancelled_call( task: asyncio.Task, discard_result: Callable | None ) -> None: """wait не переносит исключение SDK; повторная отмена не прерывает компенсацию.""" while not task.done(): with suppress(asyncio.CancelledError): await asyncio.wait([task]) completed = not task.cancelled() and task.exception() is None if completed and discard_result is not None: discard_result(task.result()) @dataclass(frozen=True) class ObjectInfo: """Поля S3 сохраняют необязательность SDK; согласованность проверяет domain.""" key: str size_bytes: int | None content_type: str | None last_modified: datetime | None def close_response(response) -> None: response.close() response.release_conn() class ObjectStream: """Закрывается даже если StreamingResponse не успел начать итерацию.""" def __init__(self, storage, response): self.storage = storage self.response = response self.deadline = time.monotonic() + storage.settings.download_timeout_seconds self.closed = False def __aiter__(self): return self async def __anext__(self): if self.closed: raise StopAsyncIteration try: remaining = self.deadline - time.monotonic() if remaining <= 0: raise TimeoutError async with asyncio.timeout(remaining): chunk = await self.storage._call( lambda: self.response.read(self.storage.settings.chunk_bytes) ) if not chunk: await self.aclose() raise StopAsyncIteration except BaseException: await self.aclose() raise return chunk async def aclose(self): if not self.closed: self.closed = True close_response(self.response) class MinioStorage: def __init__(self, settings: S3Settings) -> None: self.settings = settings # Клиент создаётся в event loop один раз, не конкурентно в worker threads. url = urlsplit(settings.endpoint) self._http = urllib3.PoolManager( timeout=urllib3.Timeout( connect=settings.connect_timeout_seconds, read=settings.read_timeout_seconds, ), retries=urllib3.Retry(total=0), maxsize=32, ) self._health_http = urllib3.PoolManager( timeout=urllib3.Timeout(connect=0.25, read=0.5), retries=urllib3.Retry(total=0), ) self._client = Minio( url.netloc, secure=url.scheme == "https", region=settings.region, access_key=settings.access_key_id_file.read_text().strip(), secret_key=settings.secret_access_key_file.read_text().strip(), http_client=self._http, ) async def _call(self, operation: Callable, *, discard_result=None): return await run_sync(operation, discard_result=discard_result) async def put_file( self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str ) -> None: def put(): with path.open("rb") as source: # Один PUT: размер части равен максимальному разрешённому файлу. self._client.put_object( self.settings.bucket, key, source, size_bytes, content_type=content_type, metadata={"sha256": sha256}, part_size=10_485_760, num_parallel_uploads=1, ) await self._call(put) async def open_object(self, key: str) -> ObjectStream: response = await self._call( lambda: self._client.get_object(self.settings.bucket, key), discard_result=close_response, ) return ObjectStream(self, response) async def stat_object(self, key: str) -> ObjectInfo: def stat(): info = self._client.stat_object(self.settings.bucket, key) return ObjectInfo(key, info.size, info.content_type, info.last_modified) return await self._call(stat) async def delete_object(self, key: str) -> None: await self._call(lambda: self._client.remove_object(self.settings.bucket, key)) async def check_available(self) -> bool: status = await self._call( lambda: self._health_http.request( "GET", self.settings.endpoint.rstrip("/") + "/minio/health/ready", ).status ) return status == 200 async def aclose(self) -> None: self._http.clear() self._health_http.clear() def create_s3_storage(settings: S3Settings) -> MinioStorage: return MinioStorage(settings)