Files
mtusi2026lab5/app/storage.py
T
2026-11-10 14:47:33 +03:00

173 lines
6.2 KiB
Python

"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway).
One long-lived aiobotocore client per API process, path-style addressing,
short connect/read timeouts, at most 2 attempts and an overall deadline
S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a
fast 503 on the attachment endpoints only. Any client-side or
network error is raised as StorageUnavailable; a missing key is
ObjectMissing. Object keys are generated by the server (see services.py);
the user's filename never becomes part of a key.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import AsyncIterator
from contextlib import AsyncExitStack
from dataclasses import dataclass
from typing import Any, BinaryIO
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from app.config import settings
logger = logging.getLogger("messenger.storage")
DOWNLOAD_CHUNK = 64 * 1024
class StorageUnavailable(Exception):
"""S3 is unreachable, timed out or refused the request."""
class ObjectMissing(Exception):
"""The key does not exist in the bucket."""
@dataclass
class StoredObject:
key: str
size: int
last_modified: Any
class ObjectStorage:
def __init__(self) -> None:
self._stack: AsyncExitStack | None = None
self._client = None
self.bucket = settings.s3_bucket
async def start(self) -> None:
self._stack = AsyncExitStack()
session = get_session()
self._client = await self._stack.enter_async_context(
session.create_client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key_id,
aws_secret_access_key=settings.s3_secret_access_key,
config=AioConfig(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
retries={"max_attempts": 2, "mode": "standard"},
max_pool_connections=settings.max_concurrent_uploads * 2 + 4,
),
)
)
async def close(self) -> None:
if self._stack is not None:
await self._stack.aclose()
self._stack = None
self._client = None
@staticmethod
def _deadline(size: int = 0):
return asyncio.timeout(
settings.s3_operation_timeout_seconds
+ size / settings.s3_put_min_bytes_per_second
)
def _require_client(self):
if self._client is None:
raise StorageUnavailable("S3 client is not started")
return self._client
async def put(
self,
key: str,
body: BinaryIO,
size: int,
content_type: str,
) -> None:
"""Upload a seekable file. SigV4 signs the payload SHA-256, so S3
itself rejects bytes that were damaged on the way."""
client = self._require_client()
body.seek(0)
try:
async with self._deadline(size):
await client.put_object(
Bucket=self.bucket,
Key=key,
Body=body,
ContentLength=size,
ContentType=content_type,
)
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]:
"""GET the object; S3 errors surface here, before any byte is sent."""
client = self._require_client()
try:
async with self._deadline():
response = await client.get_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
raise ObjectMissing(key) from exc
raise StorageUnavailable(type(exc).__name__) from exc
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
body = response["Body"]
async def chunks() -> AsyncIterator[bytes]:
try:
async for chunk in body.iter_chunks(DOWNLOAD_CHUNK):
yield chunk
finally:
body.close()
return chunks(), int(response["ContentLength"])
async def delete(self, key: str) -> None:
"""Idempotent: deleting a missing key succeeds in S3."""
client = self._require_client()
try:
async with self._deadline():
await client.delete_object(Bucket=self.bucket, Key=key)
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
async def exists(self, key: str) -> bool:
client = self._require_client()
try:
async with self._deadline():
await client.head_object(Bucket=self.bucket, Key=key)
return True
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
return False
raise StorageUnavailable(type(exc).__name__) from exc
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]:
client = self._require_client()
paginator = client.get_paginator("list_objects_v2")
try:
async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
for item in page.get("Contents", []):
yield StoredObject(item["Key"], item["Size"], item["LastModified"])
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
raise StorageUnavailable(type(exc).__name__) from exc
storage = ObjectStorage()