forked from mtusi2026/mtusi2026lab4
- compose: сервис s3 (SeaweedFS 4.46 по digest, volume s3data, внутренняя сеть storage,
web-интерфейсы только на 127.0.0.1 внутри контейнера), одноразовый s3-init;
- три пары ключей S3: admin (только bootstrap), api (Read+Write своего bucket),
reconciler (+List); bootstrap идемпотентен и проверяет приватность и права;
- миграция 0004: attachments (pending -> ready, случайный object_key), message_attachments,
post_message_v2 с проверкой вложений в той же транзакции;
- POST /chats/{id}/attachments: лимиты до чтения/декодирования, потоковая проверка PNG
(CRC + zlib до точного размера), JPEG через djpeg -strict, семафор загрузок;
- GET метаданных и /content (стриминг из S3, без redirect), readiness с S3,
HAProxy: деградированный режим без S3 через /health/ready/core;
- janitor незавершённых загрузок с grace period.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
174 lines
8.0 KiB
Python
174 lines
8.0 KiB
Python
"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
|
|
|
|
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
|
|
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
|
|
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
|
|
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
|
|
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
|
|
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
|
|
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
|
|
* No retries inside botocore (max_attempts=1): the request handler decides; an upload is not
|
|
retried behind the client's back (contract: a retry may create another object).
|
|
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
|
|
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
|
|
plain, widely supported S3 requests.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import contextlib
|
|
import hashlib
|
|
import logging
|
|
from collections.abc import AsyncIterator
|
|
from dataclasses import dataclass
|
|
|
|
from aiobotocore.config import AioConfig
|
|
from aiobotocore.session import get_session
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from .config import Settings
|
|
from .errors import Unavailable
|
|
|
|
log = logging.getLogger("messenger.storage")
|
|
|
|
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
|
|
|
|
|
|
class ObjectMissing(Exception):
|
|
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ObjectHead:
|
|
size: int
|
|
content_type: str
|
|
metadata: dict[str, str]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ListedObject:
|
|
key: str
|
|
size: int
|
|
last_modified_ts: float # POSIX seconds
|
|
|
|
|
|
class Storage:
|
|
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None):
|
|
self.bucket = settings.s3_bucket
|
|
self.timeout = settings.s3_timeout_seconds
|
|
self._endpoint = settings.s3_endpoint
|
|
self._region = settings.s3_region
|
|
self._access_key = access_key or settings.s3_access_key_id
|
|
self._secret_key = secret_key or settings.s3_secret_access_key
|
|
self._config = AioConfig(
|
|
connect_timeout=settings.s3_connect_timeout_seconds,
|
|
read_timeout=settings.s3_timeout_seconds,
|
|
retries={"total_max_attempts": 1, "mode": "standard"},
|
|
s3={"addressing_style": "path"},
|
|
signature_version="s3v4",
|
|
request_checksum_calculation="when_required",
|
|
response_checksum_validation="when_required",
|
|
max_pool_connections=settings.s3_max_connections,
|
|
)
|
|
self._stack: contextlib.AsyncExitStack | None = None
|
|
self._client = None
|
|
|
|
async def open(self) -> None:
|
|
self._stack = contextlib.AsyncExitStack()
|
|
session = get_session()
|
|
self._client = await self._stack.enter_async_context(session.create_client(
|
|
"s3", endpoint_url=self._endpoint, region_name=self._region,
|
|
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
|
|
|
|
async def close(self) -> None:
|
|
if self._stack is not None:
|
|
await self._stack.aclose()
|
|
self._stack = None
|
|
|
|
@property
|
|
def client(self):
|
|
assert self._client is not None, "Storage.open() was not called"
|
|
return self._client
|
|
|
|
async def _call(self, what: str, coro, deadline: float | None = None):
|
|
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
|
|
try:
|
|
return await asyncio.wait_for(coro, deadline or self.timeout)
|
|
except ClientError as exc:
|
|
code = str(exc.response.get("Error", {}).get("Code", ""))
|
|
if code in MISSING_CODES:
|
|
raise ObjectMissing() from None
|
|
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
|
|
raise Unavailable("Хранилище файлов временно недоступно") from None
|
|
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
|
|
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
|
|
raise Unavailable("Хранилище файлов временно недоступно") from None
|
|
|
|
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
|
|
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
|
|
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
|
|
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
|
|
await self._call("put", self.client.put_object(
|
|
Bucket=self.bucket, Key=key, Body=bytes(data), ContentType=content_type, ContentLength=len(data),
|
|
ContentMD5=md5, Metadata=metadata), deadline)
|
|
|
|
async def head(self, key: str) -> ObjectHead:
|
|
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
|
|
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
|
|
dict(response.get("Metadata", {})))
|
|
|
|
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
|
|
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
|
|
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
|
|
download instead of hanging the connection)."""
|
|
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
|
|
body = response["Body"]
|
|
size = int(response["ContentLength"])
|
|
|
|
async def chunks() -> AsyncIterator[bytes]:
|
|
try:
|
|
while True:
|
|
chunk = await self._call("get_body", body.read(64 * 1024))
|
|
if not chunk:
|
|
return
|
|
yield chunk
|
|
finally:
|
|
body.close()
|
|
return size, chunks()
|
|
|
|
async def read_all(self, key: str, limit: int) -> bytes:
|
|
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
|
|
size, chunks = await self.open_stream(key)
|
|
if size > limit:
|
|
raise Unavailable("Объект больше допустимого размера")
|
|
parts = [chunk async for chunk in chunks]
|
|
return b"".join(parts)
|
|
|
|
async def delete(self, key: str) -> None:
|
|
"""Idempotent: deleting a missing key is a success in S3."""
|
|
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
|
|
|
|
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
|
|
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
|
|
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
|
|
if start_after:
|
|
params["StartAfter"] = start_after
|
|
response = await self._call("list", self.client.list_objects_v2(**params))
|
|
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
|
|
for o in response.get("Contents", [])]
|
|
return items, bool(response.get("IsTruncated"))
|
|
|
|
async def ping(self, timeout: float = 1.0) -> bool:
|
|
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
|
|
answers, the bucket exists and the API credentials are accepted (Read permission)."""
|
|
try:
|
|
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
|
|
return True
|
|
except (Unavailable, ObjectMissing):
|
|
return False
|
|
|
|
|
|
SENTINEL_KEY = "health/sentinel"
|
|
OBJECT_PREFIX = "o/" # every attachment original lives under o/<random 128-bit hex>
|