from __future__ import annotations import base64 import logging from datetime import datetime from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from starlette.concurrency import run_in_threadpool from app import config from app.errors import ApiError logger = logging.getLogger("messenger.s3") ORIGINALS_PREFIX = "originals/" DOWNLOAD_CHUNK = 64 * 1024 class S3Unavailable(Exception): pass def _client(access_key: Optional[str], secret_key: Optional[str]): cfg = Config( signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT, retries={"max_attempts": 1, "mode": "standard"}, max_pool_connections=config.S3_POOL_MAX, request_checksum_calculation="when_required", response_checksum_validation="when_required", ) return boto3.session.Session().client( "s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=cfg, ) def _code(exc: ClientError) -> str: return exc.response.get("Error", {}).get("Code", "") class ObjectStore: """Synchronous boto3 client; async wrappers run it in the threadpool.""" def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None): self.bucket = config.S3_BUCKET self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY) @staticmethod def new_key(attachment_id) -> str: return ORIGINALS_PREFIX + attachment_id.hex def _put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: fileobj.seek(0) try: self._s3.put_object( Bucket=self.bucket, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type, ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), ) except (BotoCoreError, ClientError) as exc: logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "") raise S3Unavailable() from None async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: await run_in_threadpool(self._put, key, fileobj, size, content_type, sha256_hex) def _open(self, key: str): try: return self._s3.get_object(Bucket=self.bucket, Key=key) except ClientError as exc: logger.error("s3 get failed for existing metadata: %s", _code(exc)) raise S3Unavailable() from None except BotoCoreError as exc: logger.warning("s3 get failed: %s", type(exc).__name__) raise S3Unavailable() from None async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]: obj = await run_in_threadpool(self._open, key) body = obj["Body"] return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close def delete_sync(self, key: str) -> None: try: self._s3.delete_object(Bucket=self.bucket, Key=key) except (BotoCoreError, ClientError) as exc: raise S3Unavailable() from exc async def delete_quietly(self, key: str) -> bool: try: await run_in_threadpool(self.delete_sync, key) return True except S3Unavailable: logger.warning("compensating delete failed; sweeper will retry later") return False def list_originals(self) -> Iterator[Tuple[str, datetime]]: paginator = self._s3.get_paginator("list_objects_v2") try: for page in paginator.paginate(Bucket=self.bucket, Prefix=ORIGINALS_PREFIX, PaginationConfig={"PageSize": 1000}): for item in page.get("Contents", []): yield item["Key"], item["LastModified"] except (BotoCoreError, ClientError) as exc: raise S3Unavailable() from exc def delete_many(self, keys: List[str]) -> int: deleted = 0 for i in range(0, len(keys), 1000): batch = keys[i:i + 1000] try: out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True}) except (BotoCoreError, ClientError) as exc: raise S3Unavailable() from exc deleted += len(batch) - len(out.get("Errors", [])) return deleted def bucket_exists(self) -> bool: try: self._s3.head_bucket(Bucket=self.bucket) return True except ClientError as exc: if _code(exc) in ("404", "NoSuchBucket", "NotFound"): return False raise S3Unavailable() from exc except BotoCoreError as exc: raise S3Unavailable() from exc def create_bucket(self) -> bool: try: self._s3.create_bucket(Bucket=self.bucket) return True except ClientError as exc: if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): return False raise def storage_unavailable() -> ApiError: return ApiError(503, "unavailable", "attachment storage temporarily unavailable")