Files
mtusi2026lab4/app/s3.py
T
Чиняева f39ae06ef3 Реализовать лабораторную 4 на оценку 5
Добавить приватное S3-хранилище, вложения, presigned upload, reconciler и доказательства.
2026-11-07 01:37:21 +03:00

231 lines
7.5 KiB
Python

"""Thin, bounded S3 client wrapper for the attachment store.
The API identity has only Read/Write/List on the private bucket. The
administrative identity is used exclusively by ``app.s3_bootstrap``.
All boto3 calls are wrapped so that HTTP handlers never leak stack traces or
credentials. Large bodies are streamed; no method here reads a whole object
into memory.
"""
from __future__ import annotations
import hashlib
import threading
from datetime import datetime, timezone
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .config import settings
class S3Unavailable(Exception):
"""The object store is unreachable or rejected a request."""
class S3Missing(Exception):
"""The requested object does not exist."""
_clients: dict[bool, object] = {}
_clients_lock = threading.Lock()
def _make_client(admin: bool = False):
if admin:
access_key = _required_admin("S3_ADMIN_ACCESS_KEY_ID", "S3_ADMIN_ACCESS_KEY_ID_FILE")
secret_key = _required_admin("S3_ADMIN_SECRET_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY_FILE")
else:
access_key = settings.s3_access_key_id
secret_key = settings.s3_secret_access_key
config = Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=settings.s3_connect_timeout,
read_timeout=settings.s3_read_timeout,
retries={"max_attempts": 2, "mode": "standard"},
max_pool_connections=32,
)
return boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=config,
)
def _required_admin(name: str, fallback: str) -> str:
import os
from pathlib import Path
raw = os.environ.get(name)
if raw:
return raw
filename = os.environ.get(fallback)
if filename:
return Path(filename).read_text(encoding="utf-8").strip()
raise RuntimeError(f"missing required configuration: {name}")
def client(admin: bool = False):
with _clients_lock:
cached = _clients.get(admin)
if cached is None:
cached = _make_client(admin)
_clients[admin] = cached
return cached
def _error_code(error: ClientError) -> str:
return str(error.response.get("Error", {}).get("Code", ""))
def _missing(error: ClientError) -> bool:
return _error_code(error) in ("404", "NoSuchKey", "NoSuchBucket", "NotFound")
def ensure_bucket() -> None:
"""Idempotent private-bucket bootstrap using the administrative identity."""
s3 = client(admin=True)
try:
s3.head_bucket(Bucket=settings.s3_bucket)
return
except ClientError as error:
if _error_code(error) not in ("404", "NoSuchBucket", "NotFound"):
raise S3Unavailable("S3 head_bucket failed") from error
try:
s3.create_bucket(Bucket=settings.s3_bucket)
except ClientError as error:
code = _error_code(error)
if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
raise S3Unavailable("S3 create_bucket failed") from error
try:
s3.head_bucket(Bucket=settings.s3_bucket)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("bucket is not usable after bootstrap") from error
def check_ready() -> bool:
try:
client().head_bucket(Bucket=settings.s3_bucket)
return True
except Exception:
return False
def put_file(key: str, fileobj, content_type: str, size: int, sha256: str) -> None:
try:
client().put_object(
Bucket=settings.s3_bucket,
Key=key,
Body=fileobj,
ContentType=content_type,
ContentLength=size,
Metadata={"sha256": sha256},
)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("S3 upload failed") from error
def delete(key: str) -> None:
try:
client().delete_object(Bucket=settings.s3_bucket, Key=key)
except (BotoCoreError, ClientError):
# Compensation is best effort; the reconciler is the backstop.
return
def head(key: str) -> dict | None:
try:
return client().head_object(Bucket=settings.s3_bucket, Key=key)
except ClientError as error:
if _missing(error):
return None
raise S3Unavailable("S3 head_object failed") from error
except BotoCoreError as error:
raise S3Unavailable("S3 head_object failed") from error
def open_stream(key: str):
"""Return a streaming body. The caller must close it."""
try:
return client().get_object(Bucket=settings.s3_bucket, Key=key)["Body"]
except ClientError as error:
if _missing(error):
raise S3Missing("object is missing") from error
raise S3Unavailable("S3 get_object failed") from error
except BotoCoreError as error:
raise S3Unavailable("S3 get_object failed") from error
def stream_sha256(key: str) -> tuple[int, str]:
"""Read an object once, bounded by 64 KiB chunks, and return size+digest."""
body = open_stream(key)
digest = hashlib.sha256()
total = 0
try:
for chunk in body.iter_chunks(chunk_size=64 * 1024):
total += len(chunk)
digest.update(chunk)
except (BotoCoreError, ClientError, OSError) as error:
raise S3Unavailable("S3 streaming read failed") from error
finally:
body.close()
return total, digest.hexdigest()
def presign_put(key: str, content_type: str, sha256: str, expires: int) -> str:
"""Create a short-lived presigned PUT URL on the public S3 endpoint.
The API's internal endpoint is not reachable from user clients, so the
presigner uses S3_PUBLIC_ENDPOINT. The bucket stays private: the URL is
the only grant and it expires quickly.
"""
presigner = boto3.client(
"s3",
endpoint_url=settings.s3_public_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key_id,
aws_secret_access_key=settings.s3_secret_access_key,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
try:
return presigner.generate_presigned_url(
"put_object",
Params={
"Bucket": settings.s3_bucket,
"Key": key,
"ContentType": content_type,
"Metadata": {"sha256": sha256},
},
ExpiresIn=expires,
)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("could not sign upload URL") from error
def list_objects(prefix: str = "") -> list[dict]:
"""List bucket objects with key, size and LastModified (UTC)."""
s3 = client()
result: list[dict] = []
try:
paginator = s3.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix):
for item in page.get("Contents", []):
modified = item.get("LastModified")
if isinstance(modified, datetime) and modified.tzinfo is None:
modified = modified.replace(tzinfo=timezone.utc)
result.append(
{
"key": item["Key"],
"size": int(item.get("Size", 0)),
"last_modified": modified,
}
)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("S3 list_objects failed") from error
return result