Files
mtusi2026lab5/app/objects.py
T

185 lines
7.1 KiB
Python

from __future__ import annotations
import base64
import logging
import uuid
from datetime import datetime
from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from starlette.concurrency import run_in_threadpool
from app import config
from app.errors import ApiError
logger = logging.getLogger("messenger.s3")
# Private prefixes of the single private bucket:
# staging/<attachment> full original accepted by the API before 202 (durable handoff)
# attachments/<attachment>/original published original (same bytes, written by the worker)
# attachments/<attachment>/v<N>/<variant>.* variants of pipeline version N; key = effect key, so a retry
# overwrites the same object instead of creating a second set
STAGING_PREFIX = "staging/"
FINAL_PREFIX = "attachments/"
DOWNLOAD_CHUNK = 64 * 1024
VARIANT_EXT = {"image/jpeg": "jpg", "image/webp": "webp", "image/png": "png"}
class S3Unavailable(Exception):
pass
class S3NotFound(Exception):
pass
def staging_key(attachment_id: uuid.UUID) -> str:
return STAGING_PREFIX + attachment_id.hex
def original_key(attachment_id: uuid.UUID) -> str:
return "%s%s/original" % (FINAL_PREFIX, attachment_id.hex)
def variant_key(attachment_id: uuid.UUID, pipeline_version: int, name: str, content_type: str) -> str:
return "%s%s/v%d/%s.%s" % (FINAL_PREFIX, attachment_id.hex, pipeline_version, name, VARIANT_EXT[content_type])
def _client(access_key: Optional[str], secret_key: Optional[str]):
cfg = Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=config.S3_CONNECT_TIMEOUT,
read_timeout=config.S3_READ_TIMEOUT,
retries={"max_attempts": 1, "mode": "standard"},
max_pool_connections=config.S3_POOL_MAX,
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
)
return boto3.session.Session().client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=cfg,
)
def _code(exc: ClientError) -> str:
return exc.response.get("Error", {}).get("Code", "")
class ObjectStore:
"""Synchronous boto3 client; async wrappers run it in the threadpool."""
def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None):
self.bucket = config.S3_BUCKET
self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY)
def put_sync(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
fileobj.seek(0)
try:
# x-amz-checksum-sha256: S3 rejects the PUT if the received bytes differ from what we hashed.
self._s3.put_object(
Bucket=self.bucket,
Key=key,
Body=fileobj,
ContentLength=size,
ContentType=content_type,
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
)
except (BotoCoreError, ClientError) as exc:
logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "")
raise S3Unavailable() from None
async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
await run_in_threadpool(self.put_sync, key, fileobj, size, content_type, sha256_hex)
def download_sync(self, key: str, fileobj: BinaryIO, max_bytes: int) -> int:
"""Copy an object into fileobj; raises S3NotFound for a missing key (permanent), S3Unavailable otherwise."""
try:
obj = self._s3.get_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if _code(exc) in ("NoSuchKey", "404", "NotFound"):
raise S3NotFound(key) from None
raise S3Unavailable() from None
except BotoCoreError:
raise S3Unavailable() from None
size = 0
try:
for chunk in obj["Body"].iter_chunks(DOWNLOAD_CHUNK):
size += len(chunk)
if size > max_bytes:
raise S3Unavailable()
fileobj.write(chunk)
except BotoCoreError:
raise S3Unavailable() from None
finally:
obj["Body"].close()
return size
def _open(self, key: str):
try:
return self._s3.get_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
logger.error("s3 get failed for existing metadata: %s", _code(exc))
raise S3Unavailable() from None
except BotoCoreError as exc:
logger.warning("s3 get failed: %s", type(exc).__name__)
raise S3Unavailable() from None
async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]:
obj = await run_in_threadpool(self._open, key)
body = obj["Body"]
return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close
def delete_sync(self, key: str) -> None:
try:
self._s3.delete_object(Bucket=self.bucket, Key=key)
except (BotoCoreError, ClientError) as exc:
raise S3Unavailable() from exc
async def delete_quietly(self, key: str) -> bool:
try:
await run_in_threadpool(self.delete_sync, key)
return True
except S3Unavailable:
logger.warning("compensating delete failed; sweeper will retry later")
return False
def list_prefix(self, prefix: str) -> Iterator[Tuple[str, datetime]]:
paginator = self._s3.get_paginator("list_objects_v2")
try:
for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix, PaginationConfig={"PageSize": 1000}):
for item in page.get("Contents", []):
yield item["Key"], item["LastModified"]
except (BotoCoreError, ClientError) as exc:
raise S3Unavailable() from exc
def delete_many(self, keys: List[str]) -> int:
deleted = 0
for i in range(0, len(keys), 1000):
batch = keys[i:i + 1000]
try:
out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True})
except (BotoCoreError, ClientError) as exc:
raise S3Unavailable() from exc
deleted += len(batch) - len(out.get("Errors", []))
return deleted
def create_bucket(self) -> bool:
try:
self._s3.create_bucket(Bucket=self.bucket)
return True
except ClientError as exc:
if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
return False
raise
def storage_unavailable() -> ApiError:
return ApiError(503, "unavailable", "attachment storage temporarily unavailable")