Files
mtusi2026lab5/app/s3.py
T
alenchha 633fb96920 ЛР 5 — БВТ2301 — Курлова — на 5
RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт.
2026-11-06 12:10:16 +03:00

123 lines
4.7 KiB
Python

import logging
from typing import Iterator, Optional, Tuple
import boto3
from botocore.config import Config as BotoConfig
from botocore.exceptions import BotoCoreError, ClientError
from app import config
from app.errors import unavailable
log = logging.getLogger("messenger.s3")
MISSING = ("NoSuchKey", "404", "NotFound")
class S3Storage:
def __init__(self) -> None:
self.client = None
self.probe_client = None
def open(self) -> None:
self.probe_client = self._client(1.0, 1.5)
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
def _client(self, connect: float, read: float):
return boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY_ID,
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
config=BotoConfig(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=connect,
read_timeout=read,
retries={"max_attempts": 0},
),
)
def probe(self) -> bool:
try:
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
try:
fileobj.seek(0)
self.client.put_object(
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
)
except (BotoCoreError, ClientError) as exc:
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
except BotoCoreError as exc:
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
body = response["Body"]
return body.iter_chunks(65536), int(response["ContentLength"])
def head(self, key: str):
"""LastModified of the object, or None when it does not exist."""
try:
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
raise
def exists(self, key: str) -> bool:
return self.head(key) is not None
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
def list_pages(self, prefix: str):
paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
yield page.get("Contents", [])
# ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient)
def put_file(self, key: str, path: str, content_type: str) -> None:
with open(path, "rb") as handle:
self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type)
def download(self, key: str, path: str) -> bool:
"""False when the object does not exist."""
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return False
raise
with open(path, "wb") as handle:
for chunk in response["Body"].iter_chunks(1 << 20):
handle.write(chunk)
return True
def copy(self, source_key: str, target_key: str) -> None:
"""Server-side copy: the original bytes never pass through the worker again."""
self.client.copy_object(
Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key},
)
def ready(self) -> bool:
try:
self.client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False