forked from mtusi2026/mtusi2026lab5
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались. Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker добавляются в следующих коммитах). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
178 lines
8.4 KiB
Python
178 lines
8.4 KiB
Python
"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
|
|
|
|
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
|
|
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
|
|
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
|
|
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
|
|
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
|
|
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
|
|
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
|
|
* ONE retry inside botocore (total_max_attempts=2), within the same asyncio deadline: after S3
|
|
restarts, pooled keep-alive connections to the old process are dead and the first request on
|
|
such a connection fails although S3 is up again (found by test_s3_outage_..._stop on a clean
|
|
clone). Every call here is idempotent PER KEY (PUT of the same bytes to the same server-chosen
|
|
key, GET, HEAD, DELETE), so repeating it cannot create a second attachment.
|
|
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
|
|
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
|
|
plain, widely supported S3 requests.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import contextlib
|
|
import hashlib
|
|
import logging
|
|
from collections.abc import AsyncIterator
|
|
from dataclasses import dataclass
|
|
|
|
from aiobotocore.config import AioConfig
|
|
from aiobotocore.session import get_session
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from .config import Settings
|
|
from .errors import Unavailable
|
|
|
|
log = logging.getLogger("messenger.storage")
|
|
|
|
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
|
|
|
|
|
|
class ObjectMissing(Exception):
|
|
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ObjectHead:
|
|
size: int
|
|
content_type: str
|
|
metadata: dict[str, str]
|
|
last_modified_ts: float # POSIX seconds
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ListedObject:
|
|
key: str
|
|
size: int
|
|
last_modified_ts: float # POSIX seconds
|
|
|
|
|
|
class Storage:
|
|
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None):
|
|
self.bucket = settings.s3_bucket
|
|
self.timeout = settings.s3_timeout_seconds
|
|
self._endpoint = settings.s3_endpoint
|
|
self._region = settings.s3_region
|
|
self._access_key = access_key or settings.s3_access_key_id
|
|
self._secret_key = secret_key or settings.s3_secret_access_key
|
|
self._config = AioConfig(
|
|
connect_timeout=settings.s3_connect_timeout_seconds,
|
|
read_timeout=settings.s3_timeout_seconds,
|
|
retries={"total_max_attempts": 2, "mode": "standard"},
|
|
s3={"addressing_style": "path"},
|
|
signature_version="s3v4",
|
|
request_checksum_calculation="when_required",
|
|
response_checksum_validation="when_required",
|
|
max_pool_connections=settings.s3_max_connections,
|
|
)
|
|
self._stack: contextlib.AsyncExitStack | None = None
|
|
self._client = None
|
|
|
|
async def open(self) -> None:
|
|
self._stack = contextlib.AsyncExitStack()
|
|
session = get_session()
|
|
self._client = await self._stack.enter_async_context(session.create_client(
|
|
"s3", endpoint_url=self._endpoint, region_name=self._region,
|
|
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
|
|
|
|
async def close(self) -> None:
|
|
if self._stack is not None:
|
|
await self._stack.aclose()
|
|
self._stack = None
|
|
|
|
@property
|
|
def client(self):
|
|
assert self._client is not None, "Storage.open() was not called"
|
|
return self._client
|
|
|
|
async def _call(self, what: str, coro, deadline: float | None = None):
|
|
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
|
|
try:
|
|
return await asyncio.wait_for(coro, deadline or self.timeout)
|
|
except ClientError as exc:
|
|
code = str(exc.response.get("Error", {}).get("Code", ""))
|
|
if code in MISSING_CODES:
|
|
raise ObjectMissing() from None
|
|
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
|
|
raise Unavailable("Хранилище файлов временно недоступно") from None
|
|
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
|
|
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
|
|
raise Unavailable("Хранилище файлов временно недоступно") from None
|
|
|
|
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
|
|
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
|
|
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
|
|
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
|
|
await self._call("put", self.client.put_object(
|
|
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
|
|
ContentMD5=md5, Metadata=metadata), deadline)
|
|
|
|
async def head(self, key: str) -> ObjectHead:
|
|
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
|
|
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
|
|
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
|
|
|
|
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
|
|
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
|
|
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
|
|
download instead of hanging the connection)."""
|
|
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
|
|
body = response["Body"]
|
|
size = int(response["ContentLength"])
|
|
|
|
async def chunks() -> AsyncIterator[bytes]:
|
|
try:
|
|
while True:
|
|
chunk = await self._call("get_body", body.read(64 * 1024))
|
|
if not chunk:
|
|
return
|
|
yield chunk
|
|
finally:
|
|
body.close()
|
|
return size, chunks()
|
|
|
|
async def read_all(self, key: str, limit: int) -> bytes:
|
|
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
|
|
size, chunks = await self.open_stream(key)
|
|
if size > limit:
|
|
raise Unavailable("Объект больше допустимого размера")
|
|
parts = [chunk async for chunk in chunks]
|
|
return b"".join(parts)
|
|
|
|
async def delete(self, key: str) -> None:
|
|
"""Idempotent: deleting a missing key is a success in S3."""
|
|
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
|
|
|
|
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
|
|
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
|
|
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
|
|
if start_after:
|
|
params["StartAfter"] = start_after
|
|
response = await self._call("list", self.client.list_objects_v2(**params))
|
|
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
|
|
for o in response.get("Contents", [])]
|
|
return items, bool(response.get("IsTruncated"))
|
|
|
|
async def ping(self, timeout: float = 1.0) -> bool:
|
|
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
|
|
answers, the bucket exists and the API credentials are accepted (Read permission)."""
|
|
try:
|
|
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
|
|
return True
|
|
except (Unavailable, ObjectMissing):
|
|
return False
|
|
|
|
|
|
SENTINEL_KEY = "health/sentinel"
|
|
OBJECT_PREFIX = "o/" # every attachment original lives under o/<random 128-bit hex>
|