Files
mtusi2026lab5/app/storage/s3.py
T
2026-11-07 16:10:32 +03:00

181 lines
6.0 KiB
Python

"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции."""
import asyncio
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import TypeVar
import time
from urllib.parse import urlsplit
from anyio import CancelScope
from minio import Minio
import urllib3
from app.core.s3 import S3Settings
T = TypeVar("T")
async def run_sync(
operation: Callable[..., T], *args, discard_result=None, **kwargs
) -> T:
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
try:
result = await asyncio.shield(task)
except asyncio.CancelledError:
with CancelScope(shield=True):
await settle_cancelled_call(task, discard_result)
raise
return result
async def settle_cancelled_call(
task: asyncio.Task, discard_result: Callable | None
) -> None:
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
while not task.done():
with suppress(asyncio.CancelledError):
await asyncio.wait([task])
completed = not task.cancelled() and task.exception() is None
if completed and discard_result is not None:
discard_result(task.result())
@dataclass(frozen=True)
class ObjectInfo:
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
key: str
size_bytes: int | None
content_type: str | None
last_modified: datetime | None
def close_response(response) -> None:
response.close()
response.release_conn()
class ObjectStream:
"""Закрывается даже если StreamingResponse не успел начать итерацию."""
def __init__(self, storage, response):
self.storage = storage
self.response = response
self.deadline = time.monotonic() + storage.settings.download_timeout_seconds
self.closed = False
def __aiter__(self):
return self
async def __anext__(self):
if self.closed:
raise StopAsyncIteration
try:
remaining = self.deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError
async with asyncio.timeout(remaining):
chunk = await self.storage._call(
lambda: self.response.read(self.storage.settings.chunk_bytes)
)
if not chunk:
await self.aclose()
raise StopAsyncIteration
except BaseException:
await self.aclose()
raise
return chunk
async def aclose(self):
if not self.closed:
self.closed = True
close_response(self.response)
class MinioStorage:
def __init__(self, settings: S3Settings) -> None:
self.settings = settings
# Клиент создаётся в event loop один раз, не конкурентно в worker threads.
url = urlsplit(settings.endpoint)
self._http = urllib3.PoolManager(
timeout=urllib3.Timeout(
connect=settings.connect_timeout_seconds,
read=settings.read_timeout_seconds,
),
retries=urllib3.Retry(total=0),
maxsize=32,
)
self._health_http = urllib3.PoolManager(
timeout=urllib3.Timeout(connect=0.25, read=0.5),
retries=urllib3.Retry(total=0),
)
self._client = Minio(
url.netloc,
secure=url.scheme == "https",
region=settings.region,
access_key=settings.access_key_id_file.read_text().strip(),
secret_key=settings.secret_access_key_file.read_text().strip(),
http_client=self._http,
)
async def _call(self, operation: Callable, *, discard_result=None):
return await run_sync(operation, discard_result=discard_result)
async def put_file(
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
) -> None:
def put():
with path.open("rb") as source:
# Один PUT: размер части равен максимальному разрешённому файлу.
self._client.put_object(
self.settings.bucket,
key,
source,
size_bytes,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10_485_760,
num_parallel_uploads=1,
)
await self._call(put)
async def open_object(self, key: str) -> ObjectStream:
response = await self._call(
lambda: self._client.get_object(self.settings.bucket, key),
discard_result=close_response,
)
return ObjectStream(self, response)
async def stat_object(self, key: str) -> ObjectInfo:
def stat():
info = self._client.stat_object(self.settings.bucket, key)
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
return await self._call(stat)
async def delete_object(self, key: str) -> None:
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
async def check_available(self) -> bool:
status = await self._call(
lambda: self._health_http.request(
"GET",
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
).status
)
return status == 200
async def aclose(self) -> None:
self._http.clear()
self._health_http.clear()
def create_s3_storage(settings: S3Settings) -> MinioStorage:
return MinioStorage(settings)