import logging import os from datetime import datetime from typing import BinaryIO, Iterator, Optional, Tuple, Union import boto3 from botocore.config import Config from botocore.exceptions import ClientError from .config import ( S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, S3_STAGING_BUCKET, S3_VARIANTS_BUCKET, ) from .errors import ObjectMissingError LOGGER = logging.getLogger("messenger.s3") def bucket_for_key(key: str) -> str: if key.startswith("staging/"): return S3_STAGING_BUCKET if key.startswith("original/"): return S3_BUCKET if key.startswith("variants/"): return S3_VARIANTS_BUCKET raise ValueError("Object key is outside configured storage namespaces") class ObjectStore: def __init__(self): self.client = boto3.client( "s3", endpoint_url=S3_ENDPOINT, region_name=S3_REGION, aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY, verify=os.getenv("S3_CA_FILE", True), config=Config( connect_timeout=2, read_timeout=10, retries={"max_attempts": 2, "mode": "standard"}, client_cert=( os.getenv("S3_CLIENT_CERT_FILE"), os.getenv("S3_CLIENT_KEY_FILE"), ), s3={"addressing_style": "path"}, ), ) def put( self, key: str, content: Union[bytes, BinaryIO], content_type: str, size_bytes: Optional[int] = None, ) -> None: if size_bytes is None: size_bytes = len(content) self.client.put_object( Bucket=bucket_for_key(key), Key=key, Body=content, ContentLength=size_bytes, ContentType=content_type, ) def get(self, key: str, max_bytes: int) -> bytes: try: response = self.client.get_object(Bucket=bucket_for_key(key), Key=key) except ClientError as exc: code = exc.response.get("Error", {}).get("Code") if code in ("NoSuchKey", "404", "NotFound"): raise ObjectMissingError from exc raise body = response["Body"] try: value = body.read(max_bytes + 1) finally: body.close() if len(value) > max_bytes: raise RuntimeError("Stored object exceeds configured safety limit") return value def delete(self, key: str) -> None: self.client.delete_object(Bucket=bucket_for_key(key), Key=key) def ready(self) -> bool: self.client.head_bucket(Bucket=S3_BUCKET) return True def objects(self) -> Iterator[Tuple[str, datetime]]: for bucket in (S3_STAGING_BUCKET, S3_BUCKET, S3_VARIANTS_BUCKET): pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket) for page in pages: for item in page.get("Contents", []): yield item["Key"], item["LastModified"]