forked from mtusi2026/mtusi2026lab5
181 lines
6.0 KiB
Python
181 lines
6.0 KiB
Python
"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции."""
|
|
|
|
import asyncio
|
|
from collections.abc import Callable
|
|
from contextlib import suppress
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import TypeVar
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
|
|
from anyio import CancelScope
|
|
from minio import Minio
|
|
import urllib3
|
|
|
|
from app.core.s3 import S3Settings
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
async def run_sync(
|
|
operation: Callable[..., T], *args, discard_result=None, **kwargs
|
|
) -> T:
|
|
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
|
|
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
|
|
try:
|
|
result = await asyncio.shield(task)
|
|
except asyncio.CancelledError:
|
|
with CancelScope(shield=True):
|
|
await settle_cancelled_call(task, discard_result)
|
|
raise
|
|
return result
|
|
|
|
|
|
async def settle_cancelled_call(
|
|
task: asyncio.Task, discard_result: Callable | None
|
|
) -> None:
|
|
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
|
|
while not task.done():
|
|
with suppress(asyncio.CancelledError):
|
|
await asyncio.wait([task])
|
|
completed = not task.cancelled() and task.exception() is None
|
|
if completed and discard_result is not None:
|
|
discard_result(task.result())
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ObjectInfo:
|
|
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
|
|
|
|
key: str
|
|
size_bytes: int | None
|
|
content_type: str | None
|
|
last_modified: datetime | None
|
|
|
|
|
|
def close_response(response) -> None:
|
|
response.close()
|
|
response.release_conn()
|
|
|
|
|
|
class ObjectStream:
|
|
"""Закрывается даже если StreamingResponse не успел начать итерацию."""
|
|
|
|
def __init__(self, storage, response):
|
|
self.storage = storage
|
|
self.response = response
|
|
self.deadline = time.monotonic() + storage.settings.download_timeout_seconds
|
|
self.closed = False
|
|
|
|
def __aiter__(self):
|
|
return self
|
|
|
|
async def __anext__(self):
|
|
if self.closed:
|
|
raise StopAsyncIteration
|
|
try:
|
|
remaining = self.deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError
|
|
async with asyncio.timeout(remaining):
|
|
chunk = await self.storage._call(
|
|
lambda: self.response.read(self.storage.settings.chunk_bytes)
|
|
)
|
|
if not chunk:
|
|
await self.aclose()
|
|
raise StopAsyncIteration
|
|
except BaseException:
|
|
await self.aclose()
|
|
raise
|
|
return chunk
|
|
|
|
async def aclose(self):
|
|
if not self.closed:
|
|
self.closed = True
|
|
close_response(self.response)
|
|
|
|
|
|
class MinioStorage:
|
|
def __init__(self, settings: S3Settings) -> None:
|
|
self.settings = settings
|
|
# Клиент создаётся в event loop один раз, не конкурентно в worker threads.
|
|
url = urlsplit(settings.endpoint)
|
|
self._http = urllib3.PoolManager(
|
|
timeout=urllib3.Timeout(
|
|
connect=settings.connect_timeout_seconds,
|
|
read=settings.read_timeout_seconds,
|
|
),
|
|
retries=urllib3.Retry(total=0),
|
|
maxsize=32,
|
|
)
|
|
self._health_http = urllib3.PoolManager(
|
|
timeout=urllib3.Timeout(connect=0.25, read=0.5),
|
|
retries=urllib3.Retry(total=0),
|
|
)
|
|
self._client = Minio(
|
|
url.netloc,
|
|
secure=url.scheme == "https",
|
|
region=settings.region,
|
|
access_key=settings.access_key_id_file.read_text().strip(),
|
|
secret_key=settings.secret_access_key_file.read_text().strip(),
|
|
http_client=self._http,
|
|
)
|
|
|
|
async def _call(self, operation: Callable, *, discard_result=None):
|
|
return await run_sync(operation, discard_result=discard_result)
|
|
|
|
async def put_file(
|
|
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
|
|
) -> None:
|
|
def put():
|
|
with path.open("rb") as source:
|
|
# Один PUT: размер части равен максимальному разрешённому файлу.
|
|
self._client.put_object(
|
|
self.settings.bucket,
|
|
key,
|
|
source,
|
|
size_bytes,
|
|
content_type=content_type,
|
|
metadata={"sha256": sha256},
|
|
part_size=10_485_760,
|
|
num_parallel_uploads=1,
|
|
)
|
|
|
|
await self._call(put)
|
|
|
|
async def open_object(self, key: str) -> ObjectStream:
|
|
response = await self._call(
|
|
lambda: self._client.get_object(self.settings.bucket, key),
|
|
discard_result=close_response,
|
|
)
|
|
return ObjectStream(self, response)
|
|
|
|
async def stat_object(self, key: str) -> ObjectInfo:
|
|
def stat():
|
|
info = self._client.stat_object(self.settings.bucket, key)
|
|
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
|
|
|
|
return await self._call(stat)
|
|
|
|
async def delete_object(self, key: str) -> None:
|
|
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
|
|
|
|
async def check_available(self) -> bool:
|
|
status = await self._call(
|
|
lambda: self._health_http.request(
|
|
"GET",
|
|
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
|
|
).status
|
|
)
|
|
return status == 200
|
|
|
|
async def aclose(self) -> None:
|
|
self._http.clear()
|
|
self._health_http.clear()
|
|
|
|
|
|
def create_s3_storage(settings: S3Settings) -> MinioStorage:
|
|
return MinioStorage(settings)
|