"""Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one.""" from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FutureTimeout from dataclasses import dataclass from datetime import datetime from typing import BinaryIO, List import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from .config import ObjectSettings ORIGINAL_PREFIX = 'originals/' # Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler # must still answer inside HAProxy's one-second health check. _POOL = ThreadPoolExecutor(max_workers=32, thread_name_prefix='s3') class MissingObject(Exception): """The key is not in the bucket.""" class StorageError(Exception): """The object store could not complete the call. The message stays internal.""" @dataclass(frozen=True) class StoredObject: key: str last_modified: datetime size: int @dataclass(frozen=True) class MultipartUpload: key: str upload_id: str initiated: datetime class ObjectStore: def __init__(self, settings: ObjectSettings) -> None: self._bucket = settings.bucket self._client = boto3.client( 's3', endpoint_url=settings.endpoint, region_name=settings.region, aws_access_key_id=settings.access_key, aws_secret_access_key=settings.secret_key, config=Config( signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=30, retries={'max_attempts': 1}, ), ) def ready(self) -> None: self._bounded(self._head, 0.7) def put(self, key: str, body: bytes, content_type: str) -> None: self._bounded(lambda: self._put_bytes(key, body, content_type), 30) def put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None: self._bounded(lambda: self._put_stream(key, body, size, content_type), 30) def get(self, key: str) -> bytes: return self._bounded(lambda: self._get(key), 15) def delete(self, key: str) -> None: self._bounded(lambda: self._delete(key), 8) def list_originals(self, timeout: float = 15) -> List[StoredObject]: return self._bounded(self._list_originals, timeout) def list_multipart(self) -> List[MultipartUpload]: return self._bounded(self._list_multipart, 15) def abort_multipart(self, key: str, upload_id: str) -> None: self._bounded(lambda: self._abort_multipart(key, upload_id), 8) def _bounded(self, fn, timeout: float): future = _POOL.submit(fn) try: return future.result(timeout=timeout) except FutureTimeout: raise StorageError() from None def _head(self) -> None: try: self._client.head_bucket(Bucket=self._bucket) except (BotoCoreError, ClientError): raise StorageError() from None def _put_bytes(self, key: str, body: bytes, content_type: str) -> None: try: self._client.put_object( Bucket=self._bucket, Key=key, Body=body, ContentType=content_type, ContentLength=len(body), ) except (BotoCoreError, ClientError): raise StorageError() from None def _put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None: body.seek(0) try: self._client.put_object( Bucket=self._bucket, Key=key, Body=body, ContentType=content_type, ContentLength=size, ) except (BotoCoreError, ClientError): raise StorageError() from None def _get(self, key: str) -> bytes: try: response = self._client.get_object(Bucket=self._bucket, Key=key) except ClientError as exc: code = exc.response.get('Error', {}).get('Code', '') if code in ('NoSuchKey', '404', 'NotFound'): raise MissingObject() from None raise StorageError() from None except BotoCoreError: raise StorageError() from None payload = response['Body'] try: return payload.read() finally: payload.close() def _delete(self, key: str) -> None: try: self._client.delete_object(Bucket=self._bucket, Key=key) except (BotoCoreError, ClientError): raise StorageError() from None def _list_originals(self) -> List[StoredObject]: objects: List[StoredObject] = [] token = None try: while True: kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX} if token: kwargs['ContinuationToken'] = token response = self._client.list_objects_v2(**kwargs) for item in response.get('Contents') or (): key = item['Key'] if not key.startswith(ORIGINAL_PREFIX) or key.endswith('/'): continue objects.append(StoredObject(key, item['LastModified'], int(item['Size']))) if not response.get('IsTruncated'): return objects token = response.get('NextContinuationToken') if not token: return objects except (BotoCoreError, ClientError): raise StorageError() from None def _list_multipart(self) -> List[MultipartUpload]: uploads: List[MultipartUpload] = [] key_marker = '' upload_marker = '' try: while True: kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX} if key_marker: kwargs['KeyMarker'] = key_marker if upload_marker: kwargs['UploadIdMarker'] = upload_marker response = self._client.list_multipart_uploads(**kwargs) for item in response.get('Uploads') or (): key = item['Key'] if not key.startswith(ORIGINAL_PREFIX): continue uploads.append(MultipartUpload(key, item['UploadId'], item['Initiated'])) if not response.get('IsTruncated'): return uploads key_marker = response.get('NextKeyMarker') or '' upload_marker = response.get('NextUploadIdMarker') or '' if not key_marker and not upload_marker: return uploads except ClientError as exc: code = exc.response.get('Error', {}).get('Code', '') if code in ('NotImplemented', 'AccessDenied'): return [] raise StorageError() from None except BotoCoreError: raise StorageError() from None def _abort_multipart(self, key: str, upload_id: str) -> None: try: self._client.abort_multipart_upload( Bucket=self._bucket, Key=key, UploadId=upload_id, ) except (BotoCoreError, ClientError): raise StorageError() from None