Files
mtusi2026lab5/src/messenger/storage.py
T
Alexey BodanyukandClaude Opus 5.5 5112be83cc перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через
git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации
infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались.
Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker
добавляются в следующих коммитах).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 15:19:00 +03:00

178 lines
8.4 KiB
Python

"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
* ONE retry inside botocore (total_max_attempts=2), within the same asyncio deadline: after S3
restarts, pooled keep-alive connections to the old process are dead and the first request on
such a connection fails although S3 is up again (found by test_s3_outage_..._stop on a clean
clone). Every call here is idempotent PER KEY (PUT of the same bytes to the same server-chosen
key, GET, HEAD, DELETE), so repeating it cannot create a second attachment.
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
plain, widely supported S3 requests.
"""
from __future__ import annotations
import asyncio
import base64
import contextlib
import hashlib
import logging
from collections.abc import AsyncIterator
from dataclasses import dataclass
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.storage")
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
class ObjectMissing(Exception):
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
@dataclass(frozen=True, slots=True)
class ObjectHead:
size: int
content_type: str
metadata: dict[str, str]
last_modified_ts: float # POSIX seconds
@dataclass(frozen=True, slots=True)
class ListedObject:
key: str
size: int
last_modified_ts: float # POSIX seconds
class Storage:
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None):
self.bucket = settings.s3_bucket
self.timeout = settings.s3_timeout_seconds
self._endpoint = settings.s3_endpoint
self._region = settings.s3_region
self._access_key = access_key or settings.s3_access_key_id
self._secret_key = secret_key or settings.s3_secret_access_key
self._config = AioConfig(
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_timeout_seconds,
retries={"total_max_attempts": 2, "mode": "standard"},
s3={"addressing_style": "path"},
signature_version="s3v4",
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
max_pool_connections=settings.s3_max_connections,
)
self._stack: contextlib.AsyncExitStack | None = None
self._client = None
async def open(self) -> None:
self._stack = contextlib.AsyncExitStack()
session = get_session()
self._client = await self._stack.enter_async_context(session.create_client(
"s3", endpoint_url=self._endpoint, region_name=self._region,
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
async def close(self) -> None:
if self._stack is not None:
await self._stack.aclose()
self._stack = None
@property
def client(self):
assert self._client is not None, "Storage.open() was not called"
return self._client
async def _call(self, what: str, coro, deadline: float | None = None):
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
try:
return await asyncio.wait_for(coro, deadline or self.timeout)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code in MISSING_CODES:
raise ObjectMissing() from None
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
raise Unavailable("Хранилище файлов временно недоступно") from None
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
raise Unavailable("Хранилище файлов временно недоступно") from None
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
await self._call("put", self.client.put_object(
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
ContentMD5=md5, Metadata=metadata), deadline)
async def head(self, key: str) -> ObjectHead:
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
download instead of hanging the connection)."""
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
body = response["Body"]
size = int(response["ContentLength"])
async def chunks() -> AsyncIterator[bytes]:
try:
while True:
chunk = await self._call("get_body", body.read(64 * 1024))
if not chunk:
return
yield chunk
finally:
body.close()
return size, chunks()
async def read_all(self, key: str, limit: int) -> bytes:
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
size, chunks = await self.open_stream(key)
if size > limit:
raise Unavailable("Объект больше допустимого размера")
parts = [chunk async for chunk in chunks]
return b"".join(parts)
async def delete(self, key: str) -> None:
"""Idempotent: deleting a missing key is a success in S3."""
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
if start_after:
params["StartAfter"] = start_after
response = await self._call("list", self.client.list_objects_v2(**params))
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
for o in response.get("Contents", [])]
return items, bool(response.get("IsTruncated"))
async def ping(self, timeout: float = 1.0) -> bool:
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
answers, the bucket exists and the API credentials are accepted (Read permission)."""
try:
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
return True
except (Unavailable, ObjectMissing):
return False
SENTINEL_KEY = "health/sentinel"
OBJECT_PREFIX = "o/" # every attachment original lives under o/<random 128-bit hex>