77 lines
2.4 KiB
Python
77 lines
2.4 KiB
Python
import logging
|
|
from datetime import datetime
|
|
from typing import BinaryIO, Iterator, Optional, Tuple, Union
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import ClientError
|
|
|
|
from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY
|
|
from .errors import ObjectMissingError
|
|
|
|
LOGGER = logging.getLogger("messenger.s3")
|
|
|
|
|
|
class ObjectStore:
|
|
def __init__(self):
|
|
self.client = boto3.client(
|
|
"s3",
|
|
endpoint_url=S3_ENDPOINT,
|
|
region_name=S3_REGION,
|
|
aws_access_key_id=S3_ACCESS_KEY_ID,
|
|
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
|
config=Config(
|
|
connect_timeout=2,
|
|
read_timeout=10,
|
|
retries={"max_attempts": 2, "mode": "standard"},
|
|
s3={"addressing_style": "path"},
|
|
),
|
|
)
|
|
|
|
def put(
|
|
self,
|
|
key: str,
|
|
content: Union[bytes, BinaryIO],
|
|
content_type: str,
|
|
size_bytes: Optional[int] = None,
|
|
) -> None:
|
|
if size_bytes is None:
|
|
size_bytes = len(content)
|
|
self.client.put_object(
|
|
Bucket=S3_BUCKET,
|
|
Key=key,
|
|
Body=content,
|
|
ContentLength=size_bytes,
|
|
ContentType=content_type,
|
|
)
|
|
|
|
def get(self, key: str, max_bytes: int) -> bytes:
|
|
try:
|
|
response = self.client.get_object(Bucket=S3_BUCKET, Key=key)
|
|
except ClientError as exc:
|
|
code = exc.response.get("Error", {}).get("Code")
|
|
if code in ("NoSuchKey", "404", "NotFound"):
|
|
raise ObjectMissingError from exc
|
|
raise
|
|
body = response["Body"]
|
|
try:
|
|
value = body.read(max_bytes + 1)
|
|
finally:
|
|
body.close()
|
|
if len(value) > max_bytes:
|
|
raise RuntimeError("Stored object exceeds configured safety limit")
|
|
return value
|
|
|
|
def delete(self, key: str) -> None:
|
|
self.client.delete_object(Bucket=S3_BUCKET, Key=key)
|
|
|
|
def ready(self) -> bool:
|
|
self.client.head_bucket(Bucket=S3_BUCKET)
|
|
return True
|
|
|
|
def objects(self) -> Iterator[Tuple[str, datetime]]:
|
|
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET)
|
|
for page in pages:
|
|
for item in page.get("Contents", []):
|
|
yield item["Key"], item["LastModified"]
|