Files
mtusi2026lab5/app/storage/s3.py
T

147 lines
4.7 KiB
Python

from __future__ import annotations
import asyncio
import certifi
from dataclasses import dataclass
from datetime import datetime
from typing import BinaryIO, Iterator
from io import BytesIO
from urllib.parse import urlparse
from minio import Minio
from minio.error import InvalidResponseError, S3Error
from urllib3 import PoolManager, Retry, Timeout
from urllib3.exceptions import HTTPError
from ..domain.errors import Unavailable
def _unavailable(error: Exception) -> Unavailable:
return Unavailable('объектное хранилище недоступно')
@dataclass(slots=True)
class ObjectStream:
response: object
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
try:
while chunk := self.response.read(size):
yield chunk
finally:
self.response.close()
self.response.release_conn()
@dataclass(frozen=True, slots=True)
class StoredObject:
key: str
last_modified: datetime
class S3Storage:
def __init__(
self,
endpoint: str,
access_key: str,
secret_key: str,
bucket: str,
region: str,
) -> None:
parsed = urlparse(endpoint)
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
self._client = Minio(
parsed.netloc,
access_key=access_key,
secret_key=secret_key,
secure=parsed.scheme == 'https',
region=region,
http_client=PoolManager(
maxsize=8,
timeout=Timeout(connect=2, read=5),
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
),
)
self.bucket = bucket
@staticmethod
def _translate(error: Exception) -> None:
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
raise FileNotFoundError from error
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
raise _unavailable(error) from error
raise error
async def ping(self) -> None:
try:
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
except Exception as error:
self._translate(error)
if not exists:
raise Unavailable('bucket объектного хранилища не создан')
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
source.seek(0)
try:
await asyncio.to_thread(
self._client.put_object,
self.bucket,
key,
source,
length,
content_type,
)
except Exception as error:
self._translate(error)
async def get(self, key: str) -> ObjectStream:
try:
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
except Exception as error:
self._translate(error)
return ObjectStream(response)
async def read(self, key: str, max_bytes: int) -> bytes:
stream = await self.get(key)
result = bytearray()
for chunk in stream.chunks():
result.extend(chunk)
if len(result) > max_bytes:
raise ValueError('stored object exceeds configured byte limit')
return bytes(result)
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
await self.put(key, BytesIO(content), len(content), content_type)
async def exists(self, key: str) -> bool:
try:
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
return True
except Exception as error:
try:
self._translate(error)
except FileNotFoundError:
return False
raise
async def remove(self, key: str) -> None:
try:
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
except Exception as error:
self._translate(error)
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
def collect() -> tuple[StoredObject, ...]:
return tuple(
StoredObject(item.object_name, item.last_modified)
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
)
try:
return await asyncio.to_thread(collect)
except Exception as error:
self._translate(error)