forked from mtusi2026/mtusi2026lab5
130 lines
3.9 KiB
Python
130 lines
3.9 KiB
Python
"""S3-compatible object storage client (SeaweedFS).
|
|
|
|
Only the API process uses these limited credentials; the bucket is private and
|
|
object keys are generated by the server, never derived from ``filename``. All
|
|
connectivity failures become a bounded ``503 unavailable``.
|
|
|
|
Every operation runs under a hard deadline with a small circuit breaker: a
|
|
stopped S3 container can make Docker DNS itself block for seconds, and without
|
|
this guard a storage outage would stall request handling instead of failing fast.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import concurrent.futures
|
|
import logging
|
|
import time
|
|
from typing import BinaryIO, Iterable, Optional
|
|
|
|
import boto3
|
|
from botocore.client import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from .config import settings
|
|
from .errors import ApiError
|
|
|
|
logger = logging.getLogger("messenger.storage")
|
|
|
|
_client = None
|
|
|
|
_BOTO_CONFIG = Config(
|
|
connect_timeout=2,
|
|
read_timeout=10,
|
|
retries={"max_attempts": 1, "mode": "standard"},
|
|
s3={"addressing_style": "path"},
|
|
)
|
|
|
|
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="s3")
|
|
_BREAKER_SECONDS = 5.0
|
|
_breaker_until = 0.0
|
|
|
|
|
|
def _unavailable() -> ApiError:
|
|
return ApiError(503, "unavailable", "Хранилище недоступно")
|
|
|
|
|
|
def client():
|
|
global _client
|
|
if _client is None:
|
|
_client = boto3.client(
|
|
"s3",
|
|
endpoint_url=settings.s3_endpoint,
|
|
region_name=settings.s3_region,
|
|
aws_access_key_id=settings.s3_access_key_id or None,
|
|
aws_secret_access_key=settings.s3_secret_access_key or None,
|
|
config=_BOTO_CONFIG,
|
|
)
|
|
return _client
|
|
|
|
|
|
def _run(fn, timeout: float):
|
|
"""Run an S3 call under a deadline and a shared breaker."""
|
|
global _breaker_until
|
|
if time.monotonic() < _breaker_until:
|
|
raise _unavailable()
|
|
try:
|
|
return _executor.submit(fn).result(timeout=timeout)
|
|
except (concurrent.futures.TimeoutError, BotoCoreError, ClientError, OSError):
|
|
_breaker_until = time.monotonic() + _BREAKER_SECONDS
|
|
logger.warning("S3 unavailable; operations fail fast for %.0fs", _BREAKER_SECONDS)
|
|
raise _unavailable()
|
|
|
|
|
|
def ready() -> bool:
|
|
"""Return True if the bucket is reachable (used by /health/ready)."""
|
|
try:
|
|
_run(lambda: client().head_bucket(Bucket=settings.s3_bucket), timeout=1.5)
|
|
return True
|
|
except ApiError:
|
|
return False
|
|
|
|
|
|
def put_object(key: str, fileobj: BinaryIO, content_type: str) -> None:
|
|
_run(
|
|
lambda: client().upload_fileobj(
|
|
fileobj, settings.s3_bucket, key, ExtraArgs={"ContentType": content_type}
|
|
),
|
|
timeout=30,
|
|
)
|
|
|
|
|
|
def open_object(key: str):
|
|
"""Return the boto3 get_object response (caller streams and closes Body)."""
|
|
|
|
def _get():
|
|
try:
|
|
return client().get_object(Bucket=settings.s3_bucket, Key=key)
|
|
except ClientError as exc:
|
|
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
|
raise ApiError(404, "not_found", "Объект не найден")
|
|
raise
|
|
|
|
return _run(_get, timeout=10)
|
|
|
|
|
|
def get_bytes(key: str) -> bytes:
|
|
"""Read a whole object (bounded: attachments are <= 10 MiB)."""
|
|
response = open_object(key)
|
|
body = response["Body"]
|
|
try:
|
|
return body.read()
|
|
finally:
|
|
body.close()
|
|
|
|
|
|
def delete_object(key: str) -> bool:
|
|
try:
|
|
_run(lambda: client().delete_object(Bucket=settings.s3_bucket, Key=key), timeout=10)
|
|
return True
|
|
except ApiError:
|
|
return False
|
|
|
|
|
|
def list_keys(prefix: str = "") -> Iterable[dict]:
|
|
paginator = client().get_paginator("list_objects_v2")
|
|
try:
|
|
for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix):
|
|
for item in page.get("Contents", []):
|
|
yield item
|
|
except (BotoCoreError, ClientError):
|
|
raise _unavailable()
|