207 lines
7.3 KiB
Python
207 lines
7.3 KiB
Python
"""Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from concurrent.futures import TimeoutError as FutureTimeout
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from typing import BinaryIO, List
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from .config import ObjectSettings
|
|
|
|
ORIGINAL_PREFIX = 'originals/'
|
|
|
|
# Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler
|
|
# must still answer inside HAProxy's one-second health check.
|
|
_POOL = ThreadPoolExecutor(max_workers=32, thread_name_prefix='s3')
|
|
|
|
|
|
class MissingObject(Exception):
|
|
"""The key is not in the bucket."""
|
|
|
|
|
|
class StorageError(Exception):
|
|
"""The object store could not complete the call. The message stays internal."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class StoredObject:
|
|
key: str
|
|
last_modified: datetime
|
|
size: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MultipartUpload:
|
|
key: str
|
|
upload_id: str
|
|
initiated: datetime
|
|
|
|
|
|
class ObjectStore:
|
|
def __init__(self, settings: ObjectSettings) -> None:
|
|
self._bucket = settings.bucket
|
|
self._client = boto3.client(
|
|
's3',
|
|
endpoint_url=settings.endpoint,
|
|
region_name=settings.region,
|
|
aws_access_key_id=settings.access_key,
|
|
aws_secret_access_key=settings.secret_key,
|
|
config=Config(
|
|
signature_version='s3v4',
|
|
s3={'addressing_style': 'path'},
|
|
connect_timeout=1,
|
|
read_timeout=30,
|
|
retries={'max_attempts': 1},
|
|
),
|
|
)
|
|
|
|
def ready(self) -> None:
|
|
self._bounded(self._head, 0.7)
|
|
|
|
def put(self, key: str, body: bytes, content_type: str) -> None:
|
|
self._bounded(lambda: self._put_bytes(key, body, content_type), 30)
|
|
|
|
def put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None:
|
|
self._bounded(lambda: self._put_stream(key, body, size, content_type), 30)
|
|
|
|
def get(self, key: str) -> bytes:
|
|
return self._bounded(lambda: self._get(key), 15)
|
|
|
|
def delete(self, key: str) -> None:
|
|
self._bounded(lambda: self._delete(key), 8)
|
|
|
|
def list_originals(self, timeout: float = 15) -> List[StoredObject]:
|
|
return self._bounded(self._list_originals, timeout)
|
|
|
|
def list_multipart(self) -> List[MultipartUpload]:
|
|
return self._bounded(self._list_multipart, 15)
|
|
|
|
def abort_multipart(self, key: str, upload_id: str) -> None:
|
|
self._bounded(lambda: self._abort_multipart(key, upload_id), 8)
|
|
|
|
def _bounded(self, fn, timeout: float):
|
|
future = _POOL.submit(fn)
|
|
try:
|
|
return future.result(timeout=timeout)
|
|
except FutureTimeout:
|
|
raise StorageError() from None
|
|
|
|
def _head(self) -> None:
|
|
try:
|
|
self._client.head_bucket(Bucket=self._bucket)
|
|
except (BotoCoreError, ClientError):
|
|
raise StorageError() from None
|
|
|
|
def _put_bytes(self, key: str, body: bytes, content_type: str) -> None:
|
|
try:
|
|
self._client.put_object(
|
|
Bucket=self._bucket,
|
|
Key=key,
|
|
Body=body,
|
|
ContentType=content_type,
|
|
ContentLength=len(body),
|
|
)
|
|
except (BotoCoreError, ClientError):
|
|
raise StorageError() from None
|
|
|
|
def _put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None:
|
|
body.seek(0)
|
|
try:
|
|
self._client.put_object(
|
|
Bucket=self._bucket,
|
|
Key=key,
|
|
Body=body,
|
|
ContentType=content_type,
|
|
ContentLength=size,
|
|
)
|
|
except (BotoCoreError, ClientError):
|
|
raise StorageError() from None
|
|
|
|
def _get(self, key: str) -> bytes:
|
|
try:
|
|
response = self._client.get_object(Bucket=self._bucket, Key=key)
|
|
except ClientError as exc:
|
|
code = exc.response.get('Error', {}).get('Code', '')
|
|
if code in ('NoSuchKey', '404', 'NotFound'):
|
|
raise MissingObject() from None
|
|
raise StorageError() from None
|
|
except BotoCoreError:
|
|
raise StorageError() from None
|
|
payload = response['Body']
|
|
try:
|
|
return payload.read()
|
|
finally:
|
|
payload.close()
|
|
|
|
def _delete(self, key: str) -> None:
|
|
try:
|
|
self._client.delete_object(Bucket=self._bucket, Key=key)
|
|
except (BotoCoreError, ClientError):
|
|
raise StorageError() from None
|
|
|
|
def _list_originals(self) -> List[StoredObject]:
|
|
objects: List[StoredObject] = []
|
|
token = None
|
|
try:
|
|
while True:
|
|
kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX}
|
|
if token:
|
|
kwargs['ContinuationToken'] = token
|
|
response = self._client.list_objects_v2(**kwargs)
|
|
for item in response.get('Contents') or ():
|
|
key = item['Key']
|
|
if not key.startswith(ORIGINAL_PREFIX) or key.endswith('/'):
|
|
continue
|
|
objects.append(StoredObject(key, item['LastModified'], int(item['Size'])))
|
|
if not response.get('IsTruncated'):
|
|
return objects
|
|
token = response.get('NextContinuationToken')
|
|
if not token:
|
|
return objects
|
|
except (BotoCoreError, ClientError):
|
|
raise StorageError() from None
|
|
|
|
def _list_multipart(self) -> List[MultipartUpload]:
|
|
uploads: List[MultipartUpload] = []
|
|
key_marker = ''
|
|
upload_marker = ''
|
|
try:
|
|
while True:
|
|
kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX}
|
|
if key_marker:
|
|
kwargs['KeyMarker'] = key_marker
|
|
if upload_marker:
|
|
kwargs['UploadIdMarker'] = upload_marker
|
|
response = self._client.list_multipart_uploads(**kwargs)
|
|
for item in response.get('Uploads') or ():
|
|
key = item['Key']
|
|
if not key.startswith(ORIGINAL_PREFIX):
|
|
continue
|
|
uploads.append(MultipartUpload(key, item['UploadId'], item['Initiated']))
|
|
if not response.get('IsTruncated'):
|
|
return uploads
|
|
key_marker = response.get('NextKeyMarker') or ''
|
|
upload_marker = response.get('NextUploadIdMarker') or ''
|
|
if not key_marker and not upload_marker:
|
|
return uploads
|
|
except ClientError as exc:
|
|
code = exc.response.get('Error', {}).get('Code', '')
|
|
if code in ('NotImplemented', 'AccessDenied'):
|
|
return []
|
|
raise StorageError() from None
|
|
except BotoCoreError:
|
|
raise StorageError() from None
|
|
|
|
def _abort_multipart(self, key: str, upload_id: str) -> None:
|
|
try:
|
|
self._client.abort_multipart_upload(
|
|
Bucket=self._bucket,
|
|
Key=key,
|
|
UploadId=upload_id,
|
|
)
|
|
except (BotoCoreError, ClientError):
|
|
raise StorageError() from None
|