forked from mtusi2026/mtusi2026lab4
154 lines
6.1 KiB
Python
154 lines
6.1 KiB
Python
"""Приватное S3-хранилище (SeaweedFS) для байтов вложений.
|
|
|
|
API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket
|
|
(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер,
|
|
имя файла пользователя в ключ не попадает.
|
|
"""
|
|
import base64
|
|
import logging
|
|
from datetime import datetime
|
|
from typing import BinaryIO, Iterator
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
import settings
|
|
|
|
log = logging.getLogger("messenger.storage")
|
|
|
|
ORIGINALS_PREFIX = "originals/"
|
|
READ_CHUNK = 64 * 1024
|
|
|
|
|
|
class StorageUnavailable(Exception):
|
|
"""S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей."""
|
|
|
|
|
|
class ObjectMissing(Exception):
|
|
"""Объекта нет в bucket (NoSuchKey / 404)."""
|
|
|
|
|
|
def _make_client(connect_timeout: float, read_timeout: float, attempts: int):
|
|
return boto3.client(
|
|
"s3",
|
|
endpoint_url=settings.S3_ENDPOINT,
|
|
region_name=settings.S3_REGION,
|
|
aws_access_key_id=settings.S3_ACCESS_KEY_ID or None,
|
|
aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None,
|
|
config=Config(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"}, # http://s3:8333/<bucket>/<key>
|
|
connect_timeout=connect_timeout,
|
|
read_timeout=read_timeout,
|
|
retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую
|
|
max_pool_connections=settings.S3_POOL_MAX,
|
|
# Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно.
|
|
request_checksum_calculation="when_required",
|
|
response_checksum_validation="when_required",
|
|
),
|
|
)
|
|
|
|
|
|
_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS)
|
|
# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor.
|
|
_quick_client = _make_client(1, 2, 1)
|
|
|
|
|
|
def _is_missing(exc: ClientError) -> bool:
|
|
code = exc.response.get("Error", {}).get("Code", "")
|
|
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
|
return code in ("NoSuchKey", "404", "NotFound") or status == 404
|
|
|
|
|
|
def _unavailable(action: str, exc: Exception) -> StorageUnavailable:
|
|
detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else ""
|
|
log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail)
|
|
return StorageUnavailable()
|
|
|
|
|
|
def new_key(object_id: str) -> str:
|
|
return ORIGINALS_PREFIX + object_id
|
|
|
|
|
|
def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
|
|
"""Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект."""
|
|
body.seek(0)
|
|
try:
|
|
_client.put_object(
|
|
Bucket=settings.S3_BUCKET,
|
|
Key=key,
|
|
Body=body,
|
|
ContentLength=size,
|
|
ContentType=content_type,
|
|
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
|
|
)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise _unavailable("put", exc) from None
|
|
|
|
|
|
def open_read(key: str):
|
|
"""GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после."""
|
|
try:
|
|
return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"]
|
|
except ClientError as exc:
|
|
if _is_missing(exc):
|
|
raise ObjectMissing(key) from None
|
|
raise _unavailable("get", exc) from None
|
|
except BotoCoreError as exc:
|
|
raise _unavailable("get", exc) from None
|
|
|
|
|
|
def iter_chunks(stream) -> Iterator[bytes]:
|
|
"""Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание."""
|
|
try:
|
|
while True:
|
|
chunk = stream.read(READ_CHUNK)
|
|
if not chunk:
|
|
return
|
|
yield chunk
|
|
except (BotoCoreError, OSError) as exc:
|
|
# Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length).
|
|
log.warning("S3 stream interrupted: %s", type(exc).__name__)
|
|
raise
|
|
finally:
|
|
stream.close()
|
|
|
|
|
|
def delete(key: str, quick: bool = False) -> None:
|
|
"""Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа."""
|
|
try:
|
|
(_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise _unavailable("delete", exc) from None
|
|
|
|
|
|
def exists(key: str) -> bool:
|
|
try:
|
|
_client.head_object(Bucket=settings.S3_BUCKET, Key=key)
|
|
return True
|
|
except ClientError as exc:
|
|
if _is_missing(exc):
|
|
return False
|
|
raise _unavailable("head", exc) from None
|
|
except BotoCoreError as exc:
|
|
raise _unavailable("head", exc) from None
|
|
|
|
|
|
def list_objects(prefix: str = ORIGINALS_PREFIX) -> Iterator[tuple[str, datetime, int]]:
|
|
"""Все объекты под prefix: (key, last_modified, size). Постранично, по 1000."""
|
|
paginator = _client.get_paginator("list_objects_v2")
|
|
try:
|
|
for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix):
|
|
for item in page.get("Contents", []):
|
|
yield item["Key"], item["LastModified"], item["Size"]
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise _unavailable("list", exc) from None
|
|
|
|
|
|
def ping() -> None:
|
|
try:
|
|
_client.head_bucket(Bucket=settings.S3_BUCKET)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise _unavailable("head_bucket", exc) from None
|