Добавить приватное S3-хранилище, вложения, presigned upload, reconciler и доказательства.
231 lines
7.5 KiB
Python
231 lines
7.5 KiB
Python
"""Thin, bounded S3 client wrapper for the attachment store.
|
|
|
|
The API identity has only Read/Write/List on the private bucket. The
|
|
administrative identity is used exclusively by ``app.s3_bootstrap``.
|
|
|
|
All boto3 calls are wrapped so that HTTP handlers never leak stack traces or
|
|
credentials. Large bodies are streamed; no method here reads a whole object
|
|
into memory.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import threading
|
|
from datetime import datetime, timezone
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from .config import settings
|
|
|
|
|
|
class S3Unavailable(Exception):
|
|
"""The object store is unreachable or rejected a request."""
|
|
|
|
|
|
class S3Missing(Exception):
|
|
"""The requested object does not exist."""
|
|
|
|
|
|
_clients: dict[bool, object] = {}
|
|
_clients_lock = threading.Lock()
|
|
|
|
|
|
def _make_client(admin: bool = False):
|
|
if admin:
|
|
access_key = _required_admin("S3_ADMIN_ACCESS_KEY_ID", "S3_ADMIN_ACCESS_KEY_ID_FILE")
|
|
secret_key = _required_admin("S3_ADMIN_SECRET_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY_FILE")
|
|
else:
|
|
access_key = settings.s3_access_key_id
|
|
secret_key = settings.s3_secret_access_key
|
|
config = Config(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"},
|
|
connect_timeout=settings.s3_connect_timeout,
|
|
read_timeout=settings.s3_read_timeout,
|
|
retries={"max_attempts": 2, "mode": "standard"},
|
|
max_pool_connections=32,
|
|
)
|
|
return boto3.client(
|
|
"s3",
|
|
endpoint_url=settings.s3_endpoint,
|
|
region_name=settings.s3_region,
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
config=config,
|
|
)
|
|
|
|
|
|
def _required_admin(name: str, fallback: str) -> str:
|
|
import os
|
|
from pathlib import Path
|
|
|
|
raw = os.environ.get(name)
|
|
if raw:
|
|
return raw
|
|
filename = os.environ.get(fallback)
|
|
if filename:
|
|
return Path(filename).read_text(encoding="utf-8").strip()
|
|
raise RuntimeError(f"missing required configuration: {name}")
|
|
|
|
|
|
def client(admin: bool = False):
|
|
with _clients_lock:
|
|
cached = _clients.get(admin)
|
|
if cached is None:
|
|
cached = _make_client(admin)
|
|
_clients[admin] = cached
|
|
return cached
|
|
|
|
|
|
def _error_code(error: ClientError) -> str:
|
|
return str(error.response.get("Error", {}).get("Code", ""))
|
|
|
|
|
|
def _missing(error: ClientError) -> bool:
|
|
return _error_code(error) in ("404", "NoSuchKey", "NoSuchBucket", "NotFound")
|
|
|
|
|
|
def ensure_bucket() -> None:
|
|
"""Idempotent private-bucket bootstrap using the administrative identity."""
|
|
s3 = client(admin=True)
|
|
try:
|
|
s3.head_bucket(Bucket=settings.s3_bucket)
|
|
return
|
|
except ClientError as error:
|
|
if _error_code(error) not in ("404", "NoSuchBucket", "NotFound"):
|
|
raise S3Unavailable("S3 head_bucket failed") from error
|
|
try:
|
|
s3.create_bucket(Bucket=settings.s3_bucket)
|
|
except ClientError as error:
|
|
code = _error_code(error)
|
|
if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
|
raise S3Unavailable("S3 create_bucket failed") from error
|
|
try:
|
|
s3.head_bucket(Bucket=settings.s3_bucket)
|
|
except (BotoCoreError, ClientError) as error:
|
|
raise S3Unavailable("bucket is not usable after bootstrap") from error
|
|
|
|
|
|
def check_ready() -> bool:
|
|
try:
|
|
client().head_bucket(Bucket=settings.s3_bucket)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def put_file(key: str, fileobj, content_type: str, size: int, sha256: str) -> None:
|
|
try:
|
|
client().put_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=key,
|
|
Body=fileobj,
|
|
ContentType=content_type,
|
|
ContentLength=size,
|
|
Metadata={"sha256": sha256},
|
|
)
|
|
except (BotoCoreError, ClientError) as error:
|
|
raise S3Unavailable("S3 upload failed") from error
|
|
|
|
|
|
def delete(key: str) -> None:
|
|
try:
|
|
client().delete_object(Bucket=settings.s3_bucket, Key=key)
|
|
except (BotoCoreError, ClientError):
|
|
# Compensation is best effort; the reconciler is the backstop.
|
|
return
|
|
|
|
|
|
def head(key: str) -> dict | None:
|
|
try:
|
|
return client().head_object(Bucket=settings.s3_bucket, Key=key)
|
|
except ClientError as error:
|
|
if _missing(error):
|
|
return None
|
|
raise S3Unavailable("S3 head_object failed") from error
|
|
except BotoCoreError as error:
|
|
raise S3Unavailable("S3 head_object failed") from error
|
|
|
|
|
|
def open_stream(key: str):
|
|
"""Return a streaming body. The caller must close it."""
|
|
try:
|
|
return client().get_object(Bucket=settings.s3_bucket, Key=key)["Body"]
|
|
except ClientError as error:
|
|
if _missing(error):
|
|
raise S3Missing("object is missing") from error
|
|
raise S3Unavailable("S3 get_object failed") from error
|
|
except BotoCoreError as error:
|
|
raise S3Unavailable("S3 get_object failed") from error
|
|
|
|
|
|
def stream_sha256(key: str) -> tuple[int, str]:
|
|
"""Read an object once, bounded by 64 KiB chunks, and return size+digest."""
|
|
body = open_stream(key)
|
|
digest = hashlib.sha256()
|
|
total = 0
|
|
try:
|
|
for chunk in body.iter_chunks(chunk_size=64 * 1024):
|
|
total += len(chunk)
|
|
digest.update(chunk)
|
|
except (BotoCoreError, ClientError, OSError) as error:
|
|
raise S3Unavailable("S3 streaming read failed") from error
|
|
finally:
|
|
body.close()
|
|
return total, digest.hexdigest()
|
|
|
|
|
|
def presign_put(key: str, content_type: str, sha256: str, expires: int) -> str:
|
|
"""Create a short-lived presigned PUT URL on the public S3 endpoint.
|
|
|
|
The API's internal endpoint is not reachable from user clients, so the
|
|
presigner uses S3_PUBLIC_ENDPOINT. The bucket stays private: the URL is
|
|
the only grant and it expires quickly.
|
|
"""
|
|
presigner = boto3.client(
|
|
"s3",
|
|
endpoint_url=settings.s3_public_endpoint,
|
|
region_name=settings.s3_region,
|
|
aws_access_key_id=settings.s3_access_key_id,
|
|
aws_secret_access_key=settings.s3_secret_access_key,
|
|
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
|
)
|
|
try:
|
|
return presigner.generate_presigned_url(
|
|
"put_object",
|
|
Params={
|
|
"Bucket": settings.s3_bucket,
|
|
"Key": key,
|
|
"ContentType": content_type,
|
|
"Metadata": {"sha256": sha256},
|
|
},
|
|
ExpiresIn=expires,
|
|
)
|
|
except (BotoCoreError, ClientError) as error:
|
|
raise S3Unavailable("could not sign upload URL") from error
|
|
|
|
|
|
def list_objects(prefix: str = "") -> list[dict]:
|
|
"""List bucket objects with key, size and LastModified (UTC)."""
|
|
s3 = client()
|
|
result: list[dict] = []
|
|
try:
|
|
paginator = s3.get_paginator("list_objects_v2")
|
|
for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix):
|
|
for item in page.get("Contents", []):
|
|
modified = item.get("LastModified")
|
|
if isinstance(modified, datetime) and modified.tzinfo is None:
|
|
modified = modified.replace(tzinfo=timezone.utc)
|
|
result.append(
|
|
{
|
|
"key": item["Key"],
|
|
"size": int(item.get("Size", 0)),
|
|
"last_modified": modified,
|
|
}
|
|
)
|
|
except (BotoCoreError, ClientError) as error:
|
|
raise S3Unavailable("S3 list_objects failed") from error
|
|
return result
|