forked from mtusi2026/mtusi2026lab7
102 lines
3.1 KiB
Python
102 lines
3.1 KiB
Python
import logging
|
|
import os
|
|
from datetime import datetime
|
|
from typing import BinaryIO, Iterator, Optional, Tuple, Union
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import ClientError
|
|
|
|
from .config import (
|
|
S3_ACCESS_KEY_ID,
|
|
S3_BUCKET,
|
|
S3_ENDPOINT,
|
|
S3_REGION,
|
|
S3_SECRET_ACCESS_KEY,
|
|
S3_STAGING_BUCKET,
|
|
S3_VARIANTS_BUCKET,
|
|
)
|
|
from .errors import ObjectMissingError
|
|
|
|
LOGGER = logging.getLogger("messenger.s3")
|
|
|
|
|
|
def bucket_for_key(key: str) -> str:
|
|
if key.startswith("staging/"):
|
|
return S3_STAGING_BUCKET
|
|
if key.startswith("original/"):
|
|
return S3_BUCKET
|
|
if key.startswith("variants/"):
|
|
return S3_VARIANTS_BUCKET
|
|
raise ValueError("Object key is outside configured storage namespaces")
|
|
|
|
|
|
class ObjectStore:
|
|
def __init__(self):
|
|
self.client = boto3.client(
|
|
"s3",
|
|
endpoint_url=S3_ENDPOINT,
|
|
region_name=S3_REGION,
|
|
aws_access_key_id=S3_ACCESS_KEY_ID,
|
|
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
|
verify=os.getenv("S3_CA_FILE", True),
|
|
config=Config(
|
|
connect_timeout=2,
|
|
read_timeout=10,
|
|
retries={"max_attempts": 2, "mode": "standard"},
|
|
client_cert=(
|
|
os.getenv("S3_CLIENT_CERT_FILE"),
|
|
os.getenv("S3_CLIENT_KEY_FILE"),
|
|
),
|
|
s3={"addressing_style": "path"},
|
|
),
|
|
)
|
|
|
|
def put(
|
|
self,
|
|
key: str,
|
|
content: Union[bytes, BinaryIO],
|
|
content_type: str,
|
|
size_bytes: Optional[int] = None,
|
|
) -> None:
|
|
if size_bytes is None:
|
|
size_bytes = len(content)
|
|
self.client.put_object(
|
|
Bucket=bucket_for_key(key),
|
|
Key=key,
|
|
Body=content,
|
|
ContentLength=size_bytes,
|
|
ContentType=content_type,
|
|
)
|
|
|
|
def get(self, key: str, max_bytes: int) -> bytes:
|
|
try:
|
|
response = self.client.get_object(Bucket=bucket_for_key(key), Key=key)
|
|
except ClientError as exc:
|
|
code = exc.response.get("Error", {}).get("Code")
|
|
if code in ("NoSuchKey", "404", "NotFound"):
|
|
raise ObjectMissingError from exc
|
|
raise
|
|
body = response["Body"]
|
|
try:
|
|
value = body.read(max_bytes + 1)
|
|
finally:
|
|
body.close()
|
|
if len(value) > max_bytes:
|
|
raise RuntimeError("Stored object exceeds configured safety limit")
|
|
return value
|
|
|
|
def delete(self, key: str) -> None:
|
|
self.client.delete_object(Bucket=bucket_for_key(key), Key=key)
|
|
|
|
def ready(self) -> bool:
|
|
self.client.head_bucket(Bucket=S3_BUCKET)
|
|
return True
|
|
|
|
def objects(self) -> Iterator[Tuple[str, datetime]]:
|
|
for bucket in (S3_STAGING_BUCKET, S3_BUCKET, S3_VARIANTS_BUCKET):
|
|
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket)
|
|
for page in pages:
|
|
for item in page.get("Contents", []):
|
|
yield item["Key"], item["LastModified"]
|