Files
mtusi2026lab5/src/storage.py
T

130 lines
3.9 KiB
Python

"""S3-compatible object storage client (SeaweedFS).
Only the API process uses these limited credentials; the bucket is private and
object keys are generated by the server, never derived from ``filename``. All
connectivity failures become a bounded ``503 unavailable``.
Every operation runs under a hard deadline with a small circuit breaker: a
stopped S3 container can make Docker DNS itself block for seconds, and without
this guard a storage outage would stall request handling instead of failing fast.
"""
from __future__ import annotations
import concurrent.futures
import logging
import time
from typing import BinaryIO, Iterable, Optional
import boto3
from botocore.client import Config
from botocore.exceptions import BotoCoreError, ClientError
from .config import settings
from .errors import ApiError
logger = logging.getLogger("messenger.storage")
_client = None
_BOTO_CONFIG = Config(
connect_timeout=2,
read_timeout=10,
retries={"max_attempts": 1, "mode": "standard"},
s3={"addressing_style": "path"},
)
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="s3")
_BREAKER_SECONDS = 5.0
_breaker_until = 0.0
def _unavailable() -> ApiError:
return ApiError(503, "unavailable", "Хранилище недоступно")
def client():
global _client
if _client is None:
_client = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key_id or None,
aws_secret_access_key=settings.s3_secret_access_key or None,
config=_BOTO_CONFIG,
)
return _client
def _run(fn, timeout: float):
"""Run an S3 call under a deadline and a shared breaker."""
global _breaker_until
if time.monotonic() < _breaker_until:
raise _unavailable()
try:
return _executor.submit(fn).result(timeout=timeout)
except (concurrent.futures.TimeoutError, BotoCoreError, ClientError, OSError):
_breaker_until = time.monotonic() + _BREAKER_SECONDS
logger.warning("S3 unavailable; operations fail fast for %.0fs", _BREAKER_SECONDS)
raise _unavailable()
def ready() -> bool:
"""Return True if the bucket is reachable (used by /health/ready)."""
try:
_run(lambda: client().head_bucket(Bucket=settings.s3_bucket), timeout=1.5)
return True
except ApiError:
return False
def put_object(key: str, fileobj: BinaryIO, content_type: str) -> None:
_run(
lambda: client().upload_fileobj(
fileobj, settings.s3_bucket, key, ExtraArgs={"ContentType": content_type}
),
timeout=30,
)
def open_object(key: str):
"""Return the boto3 get_object response (caller streams and closes Body)."""
def _get():
try:
return client().get_object(Bucket=settings.s3_bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
raise ApiError(404, "not_found", "Объект не найден")
raise
return _run(_get, timeout=10)
def get_bytes(key: str) -> bytes:
"""Read a whole object (bounded: attachments are <= 10 MiB)."""
response = open_object(key)
body = response["Body"]
try:
return body.read()
finally:
body.close()
def delete_object(key: str) -> bool:
try:
_run(lambda: client().delete_object(Bucket=settings.s3_bucket, Key=key), timeout=10)
return True
except ApiError:
return False
def list_keys(prefix: str = "") -> Iterable[dict]:
paginator = client().get_paginator("list_objects_v2")
try:
for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix):
for item in page.get("Contents", []):
yield item
except (BotoCoreError, ClientError):
raise _unavailable()