Files
mtusi2026lab4/app/s3.py
T
alenchha 078a8341b5 ЛР 4: вложения в S3 (MinIO), reconciler БД<->S3, отчёт и evidence
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API
с потоковой проверкой целостности, метаданные в PostgreSQL и байты в
приватном bucket, отдельный S3-пользователь с ограниченными правами,
идемпотентный bootstrap bucket, компенсация и reconciler (трек 2),
тесты отказов, эксперименты по памяти и согласованности.
2026-11-02 16:59:17 +03:00

98 lines
3.6 KiB
Python

import logging
from typing import Iterator, Optional, Tuple
import boto3
from botocore.config import Config as BotoConfig
from botocore.exceptions import BotoCoreError, ClientError
from app import config
from app.errors import unavailable
log = logging.getLogger("messenger.s3")
MISSING = ("NoSuchKey", "404", "NotFound")
class S3Storage:
def __init__(self) -> None:
self.client = None
self.probe_client = None
def open(self) -> None:
self.probe_client = self._client(1.0, 1.5)
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
def _client(self, connect: float, read: float):
return boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY_ID,
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
config=BotoConfig(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=connect,
read_timeout=read,
retries={"max_attempts": 0},
),
)
def probe(self) -> bool:
try:
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
try:
fileobj.seek(0)
self.client.put_object(
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
)
except (BotoCoreError, ClientError) as exc:
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
except BotoCoreError as exc:
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
body = response["Body"]
return body.iter_chunks(65536), int(response["ContentLength"])
def head(self, key: str):
"""LastModified of the object, or None when it does not exist."""
try:
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
raise
def exists(self, key: str) -> bool:
return self.head(key) is not None
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
def list_pages(self, prefix: str):
paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
yield page.get("Contents", [])
def ready(self) -> bool:
try:
self.client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False