forked from mtusi2026/mtusi2026lab5
185 lines
7.1 KiB
Python
185 lines
7.1 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import logging
|
|
import uuid
|
|
from datetime import datetime
|
|
from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from app import config
|
|
from app.errors import ApiError
|
|
|
|
logger = logging.getLogger("messenger.s3")
|
|
|
|
# Private prefixes of the single private bucket:
|
|
# staging/<attachment> full original accepted by the API before 202 (durable handoff)
|
|
# attachments/<attachment>/original published original (same bytes, written by the worker)
|
|
# attachments/<attachment>/v<N>/<variant>.* variants of pipeline version N; key = effect key, so a retry
|
|
# overwrites the same object instead of creating a second set
|
|
STAGING_PREFIX = "staging/"
|
|
FINAL_PREFIX = "attachments/"
|
|
DOWNLOAD_CHUNK = 64 * 1024
|
|
VARIANT_EXT = {"image/jpeg": "jpg", "image/webp": "webp", "image/png": "png"}
|
|
|
|
|
|
class S3Unavailable(Exception):
|
|
pass
|
|
|
|
|
|
class S3NotFound(Exception):
|
|
pass
|
|
|
|
|
|
def staging_key(attachment_id: uuid.UUID) -> str:
|
|
return STAGING_PREFIX + attachment_id.hex
|
|
|
|
|
|
def original_key(attachment_id: uuid.UUID) -> str:
|
|
return "%s%s/original" % (FINAL_PREFIX, attachment_id.hex)
|
|
|
|
|
|
def variant_key(attachment_id: uuid.UUID, pipeline_version: int, name: str, content_type: str) -> str:
|
|
return "%s%s/v%d/%s.%s" % (FINAL_PREFIX, attachment_id.hex, pipeline_version, name, VARIANT_EXT[content_type])
|
|
|
|
|
|
def _client(access_key: Optional[str], secret_key: Optional[str]):
|
|
cfg = Config(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"},
|
|
connect_timeout=config.S3_CONNECT_TIMEOUT,
|
|
read_timeout=config.S3_READ_TIMEOUT,
|
|
retries={"max_attempts": 1, "mode": "standard"},
|
|
max_pool_connections=config.S3_POOL_MAX,
|
|
request_checksum_calculation="when_required",
|
|
response_checksum_validation="when_required",
|
|
)
|
|
return boto3.session.Session().client(
|
|
"s3",
|
|
endpoint_url=config.S3_ENDPOINT,
|
|
region_name=config.S3_REGION,
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
config=cfg,
|
|
)
|
|
|
|
|
|
def _code(exc: ClientError) -> str:
|
|
return exc.response.get("Error", {}).get("Code", "")
|
|
|
|
|
|
class ObjectStore:
|
|
"""Synchronous boto3 client; async wrappers run it in the threadpool."""
|
|
|
|
def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None):
|
|
self.bucket = config.S3_BUCKET
|
|
self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY)
|
|
|
|
def put_sync(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
|
|
fileobj.seek(0)
|
|
try:
|
|
# x-amz-checksum-sha256: S3 rejects the PUT if the received bytes differ from what we hashed.
|
|
self._s3.put_object(
|
|
Bucket=self.bucket,
|
|
Key=key,
|
|
Body=fileobj,
|
|
ContentLength=size,
|
|
ContentType=content_type,
|
|
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
|
|
)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "")
|
|
raise S3Unavailable() from None
|
|
|
|
async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
|
|
await run_in_threadpool(self.put_sync, key, fileobj, size, content_type, sha256_hex)
|
|
|
|
def download_sync(self, key: str, fileobj: BinaryIO, max_bytes: int) -> int:
|
|
"""Copy an object into fileobj; raises S3NotFound for a missing key (permanent), S3Unavailable otherwise."""
|
|
try:
|
|
obj = self._s3.get_object(Bucket=self.bucket, Key=key)
|
|
except ClientError as exc:
|
|
if _code(exc) in ("NoSuchKey", "404", "NotFound"):
|
|
raise S3NotFound(key) from None
|
|
raise S3Unavailable() from None
|
|
except BotoCoreError:
|
|
raise S3Unavailable() from None
|
|
size = 0
|
|
try:
|
|
for chunk in obj["Body"].iter_chunks(DOWNLOAD_CHUNK):
|
|
size += len(chunk)
|
|
if size > max_bytes:
|
|
raise S3Unavailable()
|
|
fileobj.write(chunk)
|
|
except BotoCoreError:
|
|
raise S3Unavailable() from None
|
|
finally:
|
|
obj["Body"].close()
|
|
return size
|
|
|
|
def _open(self, key: str):
|
|
try:
|
|
return self._s3.get_object(Bucket=self.bucket, Key=key)
|
|
except ClientError as exc:
|
|
logger.error("s3 get failed for existing metadata: %s", _code(exc))
|
|
raise S3Unavailable() from None
|
|
except BotoCoreError as exc:
|
|
logger.warning("s3 get failed: %s", type(exc).__name__)
|
|
raise S3Unavailable() from None
|
|
|
|
async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]:
|
|
obj = await run_in_threadpool(self._open, key)
|
|
body = obj["Body"]
|
|
return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close
|
|
|
|
def delete_sync(self, key: str) -> None:
|
|
try:
|
|
self._s3.delete_object(Bucket=self.bucket, Key=key)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise S3Unavailable() from exc
|
|
|
|
async def delete_quietly(self, key: str) -> bool:
|
|
try:
|
|
await run_in_threadpool(self.delete_sync, key)
|
|
return True
|
|
except S3Unavailable:
|
|
logger.warning("compensating delete failed; sweeper will retry later")
|
|
return False
|
|
|
|
def list_prefix(self, prefix: str) -> Iterator[Tuple[str, datetime]]:
|
|
paginator = self._s3.get_paginator("list_objects_v2")
|
|
try:
|
|
for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix, PaginationConfig={"PageSize": 1000}):
|
|
for item in page.get("Contents", []):
|
|
yield item["Key"], item["LastModified"]
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise S3Unavailable() from exc
|
|
|
|
def delete_many(self, keys: List[str]) -> int:
|
|
deleted = 0
|
|
for i in range(0, len(keys), 1000):
|
|
batch = keys[i:i + 1000]
|
|
try:
|
|
out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True})
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise S3Unavailable() from exc
|
|
deleted += len(batch) - len(out.get("Errors", []))
|
|
return deleted
|
|
|
|
def create_bucket(self) -> bool:
|
|
try:
|
|
self._s3.create_bucket(Bucket=self.bucket)
|
|
return True
|
|
except ClientError as exc:
|
|
if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
|
return False
|
|
raise
|
|
|
|
|
|
def storage_unavailable() -> ApiError:
|
|
return ApiError(503, "unavailable", "attachment storage temporarily unavailable")
|