forked from mtusi2026/mtusi2026lab4
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
98 lines
3.6 KiB
Python
98 lines
3.6 KiB
Python
import logging
|
|
from typing import Iterator, Optional, Tuple
|
|
|
|
import boto3
|
|
from botocore.config import Config as BotoConfig
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from app import config
|
|
from app.errors import unavailable
|
|
|
|
log = logging.getLogger("messenger.s3")
|
|
|
|
MISSING = ("NoSuchKey", "404", "NotFound")
|
|
|
|
|
|
class S3Storage:
|
|
def __init__(self) -> None:
|
|
self.client = None
|
|
self.probe_client = None
|
|
|
|
def open(self) -> None:
|
|
self.probe_client = self._client(1.0, 1.5)
|
|
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
|
|
|
|
def _client(self, connect: float, read: float):
|
|
return boto3.client(
|
|
"s3",
|
|
endpoint_url=config.S3_ENDPOINT,
|
|
region_name=config.S3_REGION,
|
|
aws_access_key_id=config.S3_ACCESS_KEY_ID,
|
|
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
|
|
config=BotoConfig(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"},
|
|
connect_timeout=connect,
|
|
read_timeout=read,
|
|
retries={"max_attempts": 0},
|
|
),
|
|
)
|
|
|
|
def probe(self) -> bool:
|
|
try:
|
|
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
|
|
try:
|
|
fileobj.seek(0)
|
|
self.client.put_object(
|
|
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
|
|
)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
|
|
raise unavailable("Хранилище файлов недоступно") from None
|
|
|
|
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
|
|
try:
|
|
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
|
except ClientError as exc:
|
|
if exc.response["Error"].get("Code", "") in MISSING:
|
|
return None
|
|
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
|
raise unavailable("Хранилище файлов недоступно") from None
|
|
except BotoCoreError as exc:
|
|
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
|
raise unavailable("Хранилище файлов недоступно") from None
|
|
body = response["Body"]
|
|
return body.iter_chunks(65536), int(response["ContentLength"])
|
|
|
|
def head(self, key: str):
|
|
"""LastModified of the object, or None when it does not exist."""
|
|
try:
|
|
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
|
|
except ClientError as exc:
|
|
if exc.response["Error"].get("Code", "") in MISSING:
|
|
return None
|
|
raise
|
|
|
|
def exists(self, key: str) -> bool:
|
|
return self.head(key) is not None
|
|
|
|
def delete(self, key: str) -> None:
|
|
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
|
|
|
|
def list_pages(self, prefix: str):
|
|
paginator = self.client.get_paginator("list_objects_v2")
|
|
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
|
|
yield page.get("Contents", [])
|
|
|
|
def ready(self) -> bool:
|
|
try:
|
|
self.client.head_bucket(Bucket=config.S3_BUCKET)
|
|
return True
|
|
except Exception:
|
|
return False
|