forked from mtusi2026/mtusi2026lab7
выполение на 5
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator, Optional, Tuple, Union
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .config import (
|
||||
S3_ACCESS_KEY_ID,
|
||||
S3_BUCKET,
|
||||
S3_ENDPOINT,
|
||||
S3_REGION,
|
||||
S3_SECRET_ACCESS_KEY,
|
||||
S3_STAGING_BUCKET,
|
||||
S3_VARIANTS_BUCKET,
|
||||
)
|
||||
from .errors import ObjectMissingError
|
||||
|
||||
LOGGER = logging.getLogger("messenger.s3")
|
||||
|
||||
|
||||
def bucket_for_key(key: str) -> str:
|
||||
if key.startswith("staging/"):
|
||||
return S3_STAGING_BUCKET
|
||||
if key.startswith("original/"):
|
||||
return S3_BUCKET
|
||||
if key.startswith("variants/"):
|
||||
return S3_VARIANTS_BUCKET
|
||||
raise ValueError("Object key is outside configured storage namespaces")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
region_name=S3_REGION,
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
verify=os.getenv("S3_CA_FILE", True),
|
||||
config=Config(
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
client_cert=(
|
||||
os.getenv("S3_CLIENT_CERT_FILE"),
|
||||
os.getenv("S3_CLIENT_KEY_FILE"),
|
||||
),
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
|
||||
def put(
|
||||
self,
|
||||
key: str,
|
||||
content: Union[bytes, BinaryIO],
|
||||
content_type: str,
|
||||
size_bytes: Optional[int] = None,
|
||||
) -> None:
|
||||
if size_bytes is None:
|
||||
size_bytes = len(content)
|
||||
self.client.put_object(
|
||||
Bucket=bucket_for_key(key),
|
||||
Key=key,
|
||||
Body=content,
|
||||
ContentLength=size_bytes,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def get(self, key: str, max_bytes: int) -> bytes:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=bucket_for_key(key), Key=key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
if code in ("NoSuchKey", "404", "NotFound"):
|
||||
raise ObjectMissingError from exc
|
||||
raise
|
||||
body = response["Body"]
|
||||
try:
|
||||
value = body.read(max_bytes + 1)
|
||||
finally:
|
||||
body.close()
|
||||
if len(value) > max_bytes:
|
||||
raise RuntimeError("Stored object exceeds configured safety limit")
|
||||
return value
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=bucket_for_key(key), Key=key)
|
||||
|
||||
def ready(self) -> bool:
|
||||
self.client.head_bucket(Bucket=S3_BUCKET)
|
||||
return True
|
||||
|
||||
def objects(self) -> Iterator[Tuple[str, datetime]]:
|
||||
for bucket in (S3_STAGING_BUCKET, S3_BUCKET, S3_VARIANTS_BUCKET):
|
||||
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket)
|
||||
for page in pages:
|
||||
for item in page.get("Contents", []):
|
||||
yield item["Key"], item["LastModified"]
|
||||
Reference in New Issue
Block a user