- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket, отдельные ограниченные ключи API, служебные порты закрыты; идемпотентный s3-init с проверкой приватности и прогревом записи - вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс, тип по содержимому, проверка filename, скачивание только участникам - согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper с grace и dry-run; attachment_ids в сообщениях, FK по чату - back-pressure на S3 PUT после найденного OOM SeaweedFS - тесты student-tests, evidence, REPORT.md, DEMO.md
152 lines
5.5 KiB
Python
152 lines
5.5 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import logging
|
|
from datetime import datetime
|
|
from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from app import config
|
|
from app.errors import ApiError
|
|
|
|
logger = logging.getLogger("messenger.s3")
|
|
|
|
ORIGINALS_PREFIX = "originals/"
|
|
DOWNLOAD_CHUNK = 64 * 1024
|
|
|
|
|
|
class S3Unavailable(Exception):
|
|
pass
|
|
|
|
|
|
def _client(access_key: Optional[str], secret_key: Optional[str]):
|
|
cfg = Config(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"},
|
|
connect_timeout=config.S3_CONNECT_TIMEOUT,
|
|
read_timeout=config.S3_READ_TIMEOUT,
|
|
retries={"max_attempts": 1, "mode": "standard"},
|
|
max_pool_connections=config.S3_POOL_MAX,
|
|
request_checksum_calculation="when_required",
|
|
response_checksum_validation="when_required",
|
|
)
|
|
return boto3.session.Session().client(
|
|
"s3",
|
|
endpoint_url=config.S3_ENDPOINT,
|
|
region_name=config.S3_REGION,
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
config=cfg,
|
|
)
|
|
|
|
|
|
def _code(exc: ClientError) -> str:
|
|
return exc.response.get("Error", {}).get("Code", "")
|
|
|
|
|
|
class ObjectStore:
|
|
"""Synchronous boto3 client; async wrappers run it in the threadpool."""
|
|
|
|
def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None):
|
|
self.bucket = config.S3_BUCKET
|
|
self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY)
|
|
|
|
@staticmethod
|
|
def new_key(attachment_id) -> str:
|
|
return ORIGINALS_PREFIX + attachment_id.hex
|
|
|
|
def _put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
|
|
fileobj.seek(0)
|
|
try:
|
|
self._s3.put_object(
|
|
Bucket=self.bucket,
|
|
Key=key,
|
|
Body=fileobj,
|
|
ContentLength=size,
|
|
ContentType=content_type,
|
|
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
|
|
)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "")
|
|
raise S3Unavailable() from None
|
|
|
|
async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
|
|
await run_in_threadpool(self._put, key, fileobj, size, content_type, sha256_hex)
|
|
|
|
def _open(self, key: str):
|
|
try:
|
|
return self._s3.get_object(Bucket=self.bucket, Key=key)
|
|
except ClientError as exc:
|
|
logger.error("s3 get failed for existing metadata: %s", _code(exc))
|
|
raise S3Unavailable() from None
|
|
except BotoCoreError as exc:
|
|
logger.warning("s3 get failed: %s", type(exc).__name__)
|
|
raise S3Unavailable() from None
|
|
|
|
async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]:
|
|
obj = await run_in_threadpool(self._open, key)
|
|
body = obj["Body"]
|
|
return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close
|
|
|
|
def delete_sync(self, key: str) -> None:
|
|
try:
|
|
self._s3.delete_object(Bucket=self.bucket, Key=key)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise S3Unavailable() from exc
|
|
|
|
async def delete_quietly(self, key: str) -> bool:
|
|
try:
|
|
await run_in_threadpool(self.delete_sync, key)
|
|
return True
|
|
except S3Unavailable:
|
|
logger.warning("compensating delete failed; sweeper will retry later")
|
|
return False
|
|
|
|
def list_originals(self) -> Iterator[Tuple[str, datetime]]:
|
|
paginator = self._s3.get_paginator("list_objects_v2")
|
|
try:
|
|
for page in paginator.paginate(Bucket=self.bucket, Prefix=ORIGINALS_PREFIX, PaginationConfig={"PageSize": 1000}):
|
|
for item in page.get("Contents", []):
|
|
yield item["Key"], item["LastModified"]
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise S3Unavailable() from exc
|
|
|
|
def delete_many(self, keys: List[str]) -> int:
|
|
deleted = 0
|
|
for i in range(0, len(keys), 1000):
|
|
batch = keys[i:i + 1000]
|
|
try:
|
|
out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True})
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise S3Unavailable() from exc
|
|
deleted += len(batch) - len(out.get("Errors", []))
|
|
return deleted
|
|
|
|
def bucket_exists(self) -> bool:
|
|
try:
|
|
self._s3.head_bucket(Bucket=self.bucket)
|
|
return True
|
|
except ClientError as exc:
|
|
if _code(exc) in ("404", "NoSuchBucket", "NotFound"):
|
|
return False
|
|
raise S3Unavailable() from exc
|
|
except BotoCoreError as exc:
|
|
raise S3Unavailable() from exc
|
|
|
|
def create_bucket(self) -> bool:
|
|
try:
|
|
self._s3.create_bucket(Bucket=self.bucket)
|
|
return True
|
|
except ClientError as exc:
|
|
if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
|
return False
|
|
raise
|
|
|
|
|
|
def storage_unavailable() -> ApiError:
|
|
return ApiError(503, "unavailable", "attachment storage temporarily unavailable")
|