147 lines
4.7 KiB
Python
147 lines
4.7 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import certifi
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from typing import BinaryIO, Iterator
|
|
from io import BytesIO
|
|
from urllib.parse import urlparse
|
|
|
|
from minio import Minio
|
|
from minio.error import InvalidResponseError, S3Error
|
|
from urllib3 import PoolManager, Retry, Timeout
|
|
from urllib3.exceptions import HTTPError
|
|
|
|
from ..domain.errors import Unavailable
|
|
|
|
|
|
def _unavailable(error: Exception) -> Unavailable:
|
|
return Unavailable('объектное хранилище недоступно')
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ObjectStream:
|
|
response: object
|
|
|
|
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
|
|
try:
|
|
while chunk := self.response.read(size):
|
|
yield chunk
|
|
finally:
|
|
self.response.close()
|
|
self.response.release_conn()
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class StoredObject:
|
|
key: str
|
|
last_modified: datetime
|
|
|
|
|
|
class S3Storage:
|
|
def __init__(
|
|
self,
|
|
endpoint: str,
|
|
access_key: str,
|
|
secret_key: str,
|
|
bucket: str,
|
|
region: str,
|
|
) -> None:
|
|
parsed = urlparse(endpoint)
|
|
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
|
|
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
|
|
self._client = Minio(
|
|
parsed.netloc,
|
|
access_key=access_key,
|
|
secret_key=secret_key,
|
|
secure=parsed.scheme == 'https',
|
|
region=region,
|
|
http_client=PoolManager(
|
|
maxsize=8,
|
|
timeout=Timeout(connect=2, read=5),
|
|
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
|
|
cert_reqs='CERT_REQUIRED',
|
|
ca_certs=certifi.where(),
|
|
),
|
|
)
|
|
self.bucket = bucket
|
|
|
|
@staticmethod
|
|
def _translate(error: Exception) -> None:
|
|
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
|
|
raise FileNotFoundError from error
|
|
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
|
|
raise _unavailable(error) from error
|
|
raise error
|
|
|
|
async def ping(self) -> None:
|
|
try:
|
|
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
|
|
except Exception as error:
|
|
self._translate(error)
|
|
if not exists:
|
|
raise Unavailable('bucket объектного хранилища не создан')
|
|
|
|
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
|
|
source.seek(0)
|
|
try:
|
|
await asyncio.to_thread(
|
|
self._client.put_object,
|
|
self.bucket,
|
|
key,
|
|
source,
|
|
length,
|
|
content_type,
|
|
)
|
|
except Exception as error:
|
|
self._translate(error)
|
|
|
|
async def get(self, key: str) -> ObjectStream:
|
|
try:
|
|
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
|
|
except Exception as error:
|
|
self._translate(error)
|
|
return ObjectStream(response)
|
|
|
|
async def read(self, key: str, max_bytes: int) -> bytes:
|
|
stream = await self.get(key)
|
|
result = bytearray()
|
|
for chunk in stream.chunks():
|
|
result.extend(chunk)
|
|
if len(result) > max_bytes:
|
|
raise ValueError('stored object exceeds configured byte limit')
|
|
return bytes(result)
|
|
|
|
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
|
|
await self.put(key, BytesIO(content), len(content), content_type)
|
|
|
|
async def exists(self, key: str) -> bool:
|
|
try:
|
|
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
|
|
return True
|
|
except Exception as error:
|
|
try:
|
|
self._translate(error)
|
|
except FileNotFoundError:
|
|
return False
|
|
raise
|
|
|
|
async def remove(self, key: str) -> None:
|
|
try:
|
|
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
|
|
except Exception as error:
|
|
self._translate(error)
|
|
|
|
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
|
|
def collect() -> tuple[StoredObject, ...]:
|
|
return tuple(
|
|
StoredObject(item.object_name, item.last_modified)
|
|
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
|
)
|
|
|
|
try:
|
|
return await asyncio.to_thread(collect)
|
|
except Exception as error:
|
|
self._translate(error)
|