выполение на 5

This commit is contained in:
kaktuz20000
2026-11-04 19:22:35 +03:00
parent 20afc2c790
commit 5a6fcff3ef
90 changed files with 8848 additions and 130 deletions
+101
View File
@@ -0,0 +1,101 @@
import logging
import os
from datetime import datetime
from typing import BinaryIO, Iterator, Optional, Tuple, Union
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from .config import (
S3_ACCESS_KEY_ID,
S3_BUCKET,
S3_ENDPOINT,
S3_REGION,
S3_SECRET_ACCESS_KEY,
S3_STAGING_BUCKET,
S3_VARIANTS_BUCKET,
)
from .errors import ObjectMissingError
LOGGER = logging.getLogger("messenger.s3")
def bucket_for_key(key: str) -> str:
if key.startswith("staging/"):
return S3_STAGING_BUCKET
if key.startswith("original/"):
return S3_BUCKET
if key.startswith("variants/"):
return S3_VARIANTS_BUCKET
raise ValueError("Object key is outside configured storage namespaces")
class ObjectStore:
def __init__(self):
self.client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
region_name=S3_REGION,
aws_access_key_id=S3_ACCESS_KEY_ID,
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
verify=os.getenv("S3_CA_FILE", True),
config=Config(
connect_timeout=2,
read_timeout=10,
retries={"max_attempts": 2, "mode": "standard"},
client_cert=(
os.getenv("S3_CLIENT_CERT_FILE"),
os.getenv("S3_CLIENT_KEY_FILE"),
),
s3={"addressing_style": "path"},
),
)
def put(
self,
key: str,
content: Union[bytes, BinaryIO],
content_type: str,
size_bytes: Optional[int] = None,
) -> None:
if size_bytes is None:
size_bytes = len(content)
self.client.put_object(
Bucket=bucket_for_key(key),
Key=key,
Body=content,
ContentLength=size_bytes,
ContentType=content_type,
)
def get(self, key: str, max_bytes: int) -> bytes:
try:
response = self.client.get_object(Bucket=bucket_for_key(key), Key=key)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code in ("NoSuchKey", "404", "NotFound"):
raise ObjectMissingError from exc
raise
body = response["Body"]
try:
value = body.read(max_bytes + 1)
finally:
body.close()
if len(value) > max_bytes:
raise RuntimeError("Stored object exceeds configured safety limit")
return value
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=bucket_for_key(key), Key=key)
def ready(self) -> bool:
self.client.head_bucket(Bucket=S3_BUCKET)
return True
def objects(self) -> Iterator[Tuple[str, datetime]]:
for bucket in (S3_STAGING_BUCKET, S3_BUCKET, S3_VARIANTS_BUCKET):
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket)
for page in pages:
for item in page.get("Contents", []):
yield item["Key"], item["LastModified"]