173 lines
6.2 KiB
Python
173 lines
6.2 KiB
Python
"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway).
|
|
|
|
One long-lived aiobotocore client per API process, path-style addressing,
|
|
short connect/read timeouts, at most 2 attempts and an overall deadline
|
|
S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a
|
|
fast 503 on the attachment endpoints only. Any client-side or
|
|
network error is raised as StorageUnavailable; a missing key is
|
|
ObjectMissing. Object keys are generated by the server (see services.py);
|
|
the user's filename never becomes part of a key.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import AsyncExitStack
|
|
from dataclasses import dataclass
|
|
from typing import Any, BinaryIO
|
|
|
|
from aiobotocore.config import AioConfig
|
|
from aiobotocore.session import get_session
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from app.config import settings
|
|
|
|
|
|
logger = logging.getLogger("messenger.storage")
|
|
|
|
DOWNLOAD_CHUNK = 64 * 1024
|
|
|
|
|
|
class StorageUnavailable(Exception):
|
|
"""S3 is unreachable, timed out or refused the request."""
|
|
|
|
|
|
class ObjectMissing(Exception):
|
|
"""The key does not exist in the bucket."""
|
|
|
|
|
|
@dataclass
|
|
class StoredObject:
|
|
key: str
|
|
size: int
|
|
last_modified: Any
|
|
|
|
|
|
class ObjectStorage:
|
|
def __init__(self) -> None:
|
|
self._stack: AsyncExitStack | None = None
|
|
self._client = None
|
|
self.bucket = settings.s3_bucket
|
|
|
|
async def start(self) -> None:
|
|
self._stack = AsyncExitStack()
|
|
session = get_session()
|
|
self._client = await self._stack.enter_async_context(
|
|
session.create_client(
|
|
"s3",
|
|
endpoint_url=settings.s3_endpoint,
|
|
region_name=settings.s3_region,
|
|
aws_access_key_id=settings.s3_access_key_id,
|
|
aws_secret_access_key=settings.s3_secret_access_key,
|
|
config=AioConfig(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"},
|
|
connect_timeout=settings.s3_connect_timeout_seconds,
|
|
read_timeout=settings.s3_read_timeout_seconds,
|
|
retries={"max_attempts": 2, "mode": "standard"},
|
|
max_pool_connections=settings.max_concurrent_uploads * 2 + 4,
|
|
),
|
|
)
|
|
)
|
|
|
|
async def close(self) -> None:
|
|
if self._stack is not None:
|
|
await self._stack.aclose()
|
|
self._stack = None
|
|
self._client = None
|
|
|
|
@staticmethod
|
|
def _deadline(size: int = 0):
|
|
return asyncio.timeout(
|
|
settings.s3_operation_timeout_seconds
|
|
+ size / settings.s3_put_min_bytes_per_second
|
|
)
|
|
|
|
def _require_client(self):
|
|
if self._client is None:
|
|
raise StorageUnavailable("S3 client is not started")
|
|
return self._client
|
|
|
|
async def put(
|
|
self,
|
|
key: str,
|
|
body: BinaryIO,
|
|
size: int,
|
|
content_type: str,
|
|
) -> None:
|
|
"""Upload a seekable file. SigV4 signs the payload SHA-256, so S3
|
|
itself rejects bytes that were damaged on the way."""
|
|
client = self._require_client()
|
|
body.seek(0)
|
|
try:
|
|
async with self._deadline(size):
|
|
await client.put_object(
|
|
Bucket=self.bucket,
|
|
Key=key,
|
|
Body=body,
|
|
ContentLength=size,
|
|
ContentType=content_type,
|
|
)
|
|
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
|
raise StorageUnavailable(type(exc).__name__) from exc
|
|
|
|
async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]:
|
|
"""GET the object; S3 errors surface here, before any byte is sent."""
|
|
client = self._require_client()
|
|
try:
|
|
async with self._deadline():
|
|
response = await client.get_object(Bucket=self.bucket, Key=key)
|
|
except ClientError as exc:
|
|
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
|
raise ObjectMissing(key) from exc
|
|
raise StorageUnavailable(type(exc).__name__) from exc
|
|
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
|
raise StorageUnavailable(type(exc).__name__) from exc
|
|
|
|
body = response["Body"]
|
|
|
|
async def chunks() -> AsyncIterator[bytes]:
|
|
try:
|
|
async for chunk in body.iter_chunks(DOWNLOAD_CHUNK):
|
|
yield chunk
|
|
finally:
|
|
body.close()
|
|
|
|
return chunks(), int(response["ContentLength"])
|
|
|
|
async def delete(self, key: str) -> None:
|
|
"""Idempotent: deleting a missing key succeeds in S3."""
|
|
client = self._require_client()
|
|
try:
|
|
async with self._deadline():
|
|
await client.delete_object(Bucket=self.bucket, Key=key)
|
|
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
|
raise StorageUnavailable(type(exc).__name__) from exc
|
|
|
|
async def exists(self, key: str) -> bool:
|
|
client = self._require_client()
|
|
try:
|
|
async with self._deadline():
|
|
await client.head_object(Bucket=self.bucket, Key=key)
|
|
return True
|
|
except ClientError as exc:
|
|
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
|
return False
|
|
raise StorageUnavailable(type(exc).__name__) from exc
|
|
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
|
raise StorageUnavailable(type(exc).__name__) from exc
|
|
|
|
async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]:
|
|
client = self._require_client()
|
|
paginator = client.get_paginator("list_objects_v2")
|
|
try:
|
|
async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
|
|
for item in page.get("Contents", []):
|
|
yield StoredObject(item["Key"], item["Size"], item["LastModified"])
|
|
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
|
raise StorageUnavailable(type(exc).__name__) from exc
|
|
|
|
|
|
storage = ObjectStorage()
|