from __future__ import annotations import base64 import logging import uuid from datetime import datetime from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from starlette.concurrency import run_in_threadpool from app import config from app.errors import ApiError logger = logging.getLogger("messenger.s3") # Private prefixes of the single private bucket: # staging/ full original accepted by the API before 202 (durable handoff) # attachments//original published original (same bytes, written by the worker) # attachments//v/.* variants of pipeline version N; key = effect key, so a retry # overwrites the same object instead of creating a second set STAGING_PREFIX = "staging/" FINAL_PREFIX = "attachments/" DOWNLOAD_CHUNK = 64 * 1024 VARIANT_EXT = {"image/jpeg": "jpg", "image/webp": "webp", "image/png": "png"} class S3Unavailable(Exception): pass class S3NotFound(Exception): pass def staging_key(attachment_id: uuid.UUID) -> str: return STAGING_PREFIX + attachment_id.hex def original_key(attachment_id: uuid.UUID) -> str: return "%s%s/original" % (FINAL_PREFIX, attachment_id.hex) def variant_key(attachment_id: uuid.UUID, pipeline_version: int, name: str, content_type: str) -> str: return "%s%s/v%d/%s.%s" % (FINAL_PREFIX, attachment_id.hex, pipeline_version, name, VARIANT_EXT[content_type]) def _client(access_key: Optional[str], secret_key: Optional[str]): cfg = Config( signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT, retries={"max_attempts": 1, "mode": "standard"}, max_pool_connections=config.S3_POOL_MAX, request_checksum_calculation="when_required", response_checksum_validation="when_required", ) return boto3.session.Session().client( "s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=cfg, ) def _code(exc: ClientError) -> str: return exc.response.get("Error", {}).get("Code", "") class ObjectStore: """Synchronous boto3 client; async wrappers run it in the threadpool.""" def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None): self.bucket = config.S3_BUCKET self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY) def put_sync(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: fileobj.seek(0) try: # x-amz-checksum-sha256: S3 rejects the PUT if the received bytes differ from what we hashed. self._s3.put_object( Bucket=self.bucket, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type, ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), ) except (BotoCoreError, ClientError) as exc: logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "") raise S3Unavailable() from None async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: await run_in_threadpool(self.put_sync, key, fileobj, size, content_type, sha256_hex) def download_sync(self, key: str, fileobj: BinaryIO, max_bytes: int) -> int: """Copy an object into fileobj; raises S3NotFound for a missing key (permanent), S3Unavailable otherwise.""" try: obj = self._s3.get_object(Bucket=self.bucket, Key=key) except ClientError as exc: if _code(exc) in ("NoSuchKey", "404", "NotFound"): raise S3NotFound(key) from None raise S3Unavailable() from None except BotoCoreError: raise S3Unavailable() from None size = 0 try: for chunk in obj["Body"].iter_chunks(DOWNLOAD_CHUNK): size += len(chunk) if size > max_bytes: raise S3Unavailable() fileobj.write(chunk) except BotoCoreError: raise S3Unavailable() from None finally: obj["Body"].close() return size def _open(self, key: str): try: return self._s3.get_object(Bucket=self.bucket, Key=key) except ClientError as exc: logger.error("s3 get failed for existing metadata: %s", _code(exc)) raise S3Unavailable() from None except BotoCoreError as exc: logger.warning("s3 get failed: %s", type(exc).__name__) raise S3Unavailable() from None async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]: obj = await run_in_threadpool(self._open, key) body = obj["Body"] return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close def delete_sync(self, key: str) -> None: try: self._s3.delete_object(Bucket=self.bucket, Key=key) except (BotoCoreError, ClientError) as exc: raise S3Unavailable() from exc async def delete_quietly(self, key: str) -> bool: try: await run_in_threadpool(self.delete_sync, key) return True except S3Unavailable: logger.warning("compensating delete failed; sweeper will retry later") return False def list_prefix(self, prefix: str) -> Iterator[Tuple[str, datetime]]: paginator = self._s3.get_paginator("list_objects_v2") try: for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix, PaginationConfig={"PageSize": 1000}): for item in page.get("Contents", []): yield item["Key"], item["LastModified"] except (BotoCoreError, ClientError) as exc: raise S3Unavailable() from exc def delete_many(self, keys: List[str]) -> int: deleted = 0 for i in range(0, len(keys), 1000): batch = keys[i:i + 1000] try: out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True}) except (BotoCoreError, ClientError) as exc: raise S3Unavailable() from exc deleted += len(batch) - len(out.get("Errors", [])) return deleted def create_bucket(self) -> bool: try: self._s3.create_bucket(Bucket=self.bucket) return True except ClientError as exc: if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): return False raise def storage_unavailable() -> ApiError: return ApiError(503, "unavailable", "attachment storage temporarily unavailable")