Files
2026-11-10 20:04:30 +03:00

207 lines
7.3 KiB
Python

"""Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one."""
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FutureTimeout
from dataclasses import dataclass
from datetime import datetime
from typing import BinaryIO, List
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .config import ObjectSettings
ORIGINAL_PREFIX = 'originals/'
# Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler
# must still answer inside HAProxy's one-second health check.
_POOL = ThreadPoolExecutor(max_workers=32, thread_name_prefix='s3')
class MissingObject(Exception):
"""The key is not in the bucket."""
class StorageError(Exception):
"""The object store could not complete the call. The message stays internal."""
@dataclass(frozen=True)
class StoredObject:
key: str
last_modified: datetime
size: int
@dataclass(frozen=True)
class MultipartUpload:
key: str
upload_id: str
initiated: datetime
class ObjectStore:
def __init__(self, settings: ObjectSettings) -> None:
self._bucket = settings.bucket
self._client = boto3.client(
's3',
endpoint_url=settings.endpoint,
region_name=settings.region,
aws_access_key_id=settings.access_key,
aws_secret_access_key=settings.secret_key,
config=Config(
signature_version='s3v4',
s3={'addressing_style': 'path'},
connect_timeout=1,
read_timeout=30,
retries={'max_attempts': 1},
),
)
def ready(self) -> None:
self._bounded(self._head, 0.7)
def put(self, key: str, body: bytes, content_type: str) -> None:
self._bounded(lambda: self._put_bytes(key, body, content_type), 30)
def put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None:
self._bounded(lambda: self._put_stream(key, body, size, content_type), 30)
def get(self, key: str) -> bytes:
return self._bounded(lambda: self._get(key), 15)
def delete(self, key: str) -> None:
self._bounded(lambda: self._delete(key), 8)
def list_originals(self, timeout: float = 15) -> List[StoredObject]:
return self._bounded(self._list_originals, timeout)
def list_multipart(self) -> List[MultipartUpload]:
return self._bounded(self._list_multipart, 15)
def abort_multipart(self, key: str, upload_id: str) -> None:
self._bounded(lambda: self._abort_multipart(key, upload_id), 8)
def _bounded(self, fn, timeout: float):
future = _POOL.submit(fn)
try:
return future.result(timeout=timeout)
except FutureTimeout:
raise StorageError() from None
def _head(self) -> None:
try:
self._client.head_bucket(Bucket=self._bucket)
except (BotoCoreError, ClientError):
raise StorageError() from None
def _put_bytes(self, key: str, body: bytes, content_type: str) -> None:
try:
self._client.put_object(
Bucket=self._bucket,
Key=key,
Body=body,
ContentType=content_type,
ContentLength=len(body),
)
except (BotoCoreError, ClientError):
raise StorageError() from None
def _put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None:
body.seek(0)
try:
self._client.put_object(
Bucket=self._bucket,
Key=key,
Body=body,
ContentType=content_type,
ContentLength=size,
)
except (BotoCoreError, ClientError):
raise StorageError() from None
def _get(self, key: str) -> bytes:
try:
response = self._client.get_object(Bucket=self._bucket, Key=key)
except ClientError as exc:
code = exc.response.get('Error', {}).get('Code', '')
if code in ('NoSuchKey', '404', 'NotFound'):
raise MissingObject() from None
raise StorageError() from None
except BotoCoreError:
raise StorageError() from None
payload = response['Body']
try:
return payload.read()
finally:
payload.close()
def _delete(self, key: str) -> None:
try:
self._client.delete_object(Bucket=self._bucket, Key=key)
except (BotoCoreError, ClientError):
raise StorageError() from None
def _list_originals(self) -> List[StoredObject]:
objects: List[StoredObject] = []
token = None
try:
while True:
kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX}
if token:
kwargs['ContinuationToken'] = token
response = self._client.list_objects_v2(**kwargs)
for item in response.get('Contents') or ():
key = item['Key']
if not key.startswith(ORIGINAL_PREFIX) or key.endswith('/'):
continue
objects.append(StoredObject(key, item['LastModified'], int(item['Size'])))
if not response.get('IsTruncated'):
return objects
token = response.get('NextContinuationToken')
if not token:
return objects
except (BotoCoreError, ClientError):
raise StorageError() from None
def _list_multipart(self) -> List[MultipartUpload]:
uploads: List[MultipartUpload] = []
key_marker = ''
upload_marker = ''
try:
while True:
kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX}
if key_marker:
kwargs['KeyMarker'] = key_marker
if upload_marker:
kwargs['UploadIdMarker'] = upload_marker
response = self._client.list_multipart_uploads(**kwargs)
for item in response.get('Uploads') or ():
key = item['Key']
if not key.startswith(ORIGINAL_PREFIX):
continue
uploads.append(MultipartUpload(key, item['UploadId'], item['Initiated']))
if not response.get('IsTruncated'):
return uploads
key_marker = response.get('NextKeyMarker') or ''
upload_marker = response.get('NextUploadIdMarker') or ''
if not key_marker and not upload_marker:
return uploads
except ClientError as exc:
code = exc.response.get('Error', {}).get('Code', '')
if code in ('NotImplemented', 'AccessDenied'):
return []
raise StorageError() from None
except BotoCoreError:
raise StorageError() from None
def _abort_multipart(self, key: str, upload_id: str) -> None:
try:
self._client.abort_multipart_upload(
Bucket=self._bucket,
Key=key,
UploadId=upload_id,
)
except (BotoCoreError, ClientError):
raise StorageError() from None