Files
mtusi2026lab4/app/objects.py
T
Ilya a97c35fc12 выполнение ЛР4 на 4
- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket,
  отдельные ограниченные ключи API, служебные порты закрыты;
  идемпотентный s3-init с проверкой приватности и прогревом записи
- вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс,
  тип по содержимому, проверка filename, скачивание только участникам
- согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper
  с grace и dry-run; attachment_ids в сообщениях, FK по чату
- back-pressure на S3 PUT после найденного OOM SeaweedFS
- тесты student-tests, evidence, REPORT.md, DEMO.md
2026-10-28 18:06:07 +03:00

152 lines
5.5 KiB
Python

from __future__ import annotations
import base64
import logging
from datetime import datetime
from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from starlette.concurrency import run_in_threadpool
from app import config
from app.errors import ApiError
logger = logging.getLogger("messenger.s3")
ORIGINALS_PREFIX = "originals/"
DOWNLOAD_CHUNK = 64 * 1024
class S3Unavailable(Exception):
pass
def _client(access_key: Optional[str], secret_key: Optional[str]):
cfg = Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=config.S3_CONNECT_TIMEOUT,
read_timeout=config.S3_READ_TIMEOUT,
retries={"max_attempts": 1, "mode": "standard"},
max_pool_connections=config.S3_POOL_MAX,
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
)
return boto3.session.Session().client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=cfg,
)
def _code(exc: ClientError) -> str:
return exc.response.get("Error", {}).get("Code", "")
class ObjectStore:
"""Synchronous boto3 client; async wrappers run it in the threadpool."""
def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None):
self.bucket = config.S3_BUCKET
self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY)
@staticmethod
def new_key(attachment_id) -> str:
return ORIGINALS_PREFIX + attachment_id.hex
def _put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
fileobj.seek(0)
try:
self._s3.put_object(
Bucket=self.bucket,
Key=key,
Body=fileobj,
ContentLength=size,
ContentType=content_type,
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
)
except (BotoCoreError, ClientError) as exc:
logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "")
raise S3Unavailable() from None
async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
await run_in_threadpool(self._put, key, fileobj, size, content_type, sha256_hex)
def _open(self, key: str):
try:
return self._s3.get_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
logger.error("s3 get failed for existing metadata: %s", _code(exc))
raise S3Unavailable() from None
except BotoCoreError as exc:
logger.warning("s3 get failed: %s", type(exc).__name__)
raise S3Unavailable() from None
async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]:
obj = await run_in_threadpool(self._open, key)
body = obj["Body"]
return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close
def delete_sync(self, key: str) -> None:
try:
self._s3.delete_object(Bucket=self.bucket, Key=key)
except (BotoCoreError, ClientError) as exc:
raise S3Unavailable() from exc
async def delete_quietly(self, key: str) -> bool:
try:
await run_in_threadpool(self.delete_sync, key)
return True
except S3Unavailable:
logger.warning("compensating delete failed; sweeper will retry later")
return False
def list_originals(self) -> Iterator[Tuple[str, datetime]]:
paginator = self._s3.get_paginator("list_objects_v2")
try:
for page in paginator.paginate(Bucket=self.bucket, Prefix=ORIGINALS_PREFIX, PaginationConfig={"PageSize": 1000}):
for item in page.get("Contents", []):
yield item["Key"], item["LastModified"]
except (BotoCoreError, ClientError) as exc:
raise S3Unavailable() from exc
def delete_many(self, keys: List[str]) -> int:
deleted = 0
for i in range(0, len(keys), 1000):
batch = keys[i:i + 1000]
try:
out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True})
except (BotoCoreError, ClientError) as exc:
raise S3Unavailable() from exc
deleted += len(batch) - len(out.get("Errors", []))
return deleted
def bucket_exists(self) -> bool:
try:
self._s3.head_bucket(Bucket=self.bucket)
return True
except ClientError as exc:
if _code(exc) in ("404", "NoSuchBucket", "NotFound"):
return False
raise S3Unavailable() from exc
except BotoCoreError as exc:
raise S3Unavailable() from exc
def create_bucket(self) -> bool:
try:
self._s3.create_bucket(Bucket=self.bucket)
return True
except ClientError as exc:
if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
return False
raise
def storage_unavailable() -> ApiError:
return ApiError(503, "unavailable", "attachment storage temporarily unavailable")