fix s3 storage

This commit is contained in:
2026-11-04 22:00:11 +03:00
parent 7576ee8c83
commit 5ceab4825e
10 changed files with 274 additions and 108 deletions
+10 -6
View File
@@ -20,13 +20,13 @@ async def cleanup_once(factory, storage, apply: bool) -> bool:
result = await reconcile_attachments(session, storage, dry_run=not apply)
print(json.dumps(asdict(result)), flush=True)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error('Reconciliation', error)
log_recoverable_error("Reconciliation", error)
completed = False
return completed
async def run(once: bool, apply: bool) -> None:
settings = Settings()
settings = Settings() # type: ignore
s3_settings = S3Settings()
engine = create_db_engine(settings)
factory = create_session_factory(engine)
@@ -45,12 +45,16 @@ async def run(once: bool, apply: bool) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description='Сверка журнала загрузок MinIO/БД')
parser.add_argument('--once', action='store_true')
parser.add_argument('--apply', action='store_true', help='удалять безопасные кандидаты; по умолчанию dry-run')
parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД")
parser.add_argument("--once", action="store_true")
parser.add_argument(
"--apply",
action="store_true",
help="удалять безопасные кандидаты; по умолчанию dry-run",
)
args = parser.parse_args()
asyncio.run(run(args.once, args.apply))
if __name__ == '__main__':
if __name__ == "__main__":
main()
+16 -3
View File
@@ -3,13 +3,24 @@ from pathlib import Path
from typing import Literal, Self
from pydantic import Field, SecretStr, model_validator
from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
from pydantic_settings import (
BaseSettings,
PydanticBaseSettingsSource,
SettingsConfigDict,
)
from sqlalchemy import URL
from app.core.s3 import S3Settings
ASYNC_DRIVER = "postgresql+asyncpg"
SECRETS_DIR = next(
(path for path in (Path("/run/secrets"), Path(__file__).resolve().parents[2] / "secrets") if (path / "db_password").is_file()),
(
path
for path in (
Path("/run/secrets"),
Path(__file__).resolve().parents[2] / "secrets",
)
if (path / "db_password").is_file()
),
None,
)
@@ -88,7 +99,9 @@ class Settings(BaseSettings):
@model_validator(mode="after")
def _require_database_source(self) -> Self:
if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8:
raise ValueError("Drain и graceful shutdown должны суммарно занимать менее 8 секунд")
raise ValueError(
"Drain и graceful shutdown должны суммарно занимать менее 8 секунд"
)
if self.database_url is not None:
if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER):
raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://")
+1 -2
View File
@@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import (
from app.core.config import Settings
def create_db_engine(settings: Settings) -> AsyncEngine:
return create_async_engine(
settings.get_database_url(),
@@ -25,5 +26,3 @@ def create_db_engine(settings: Settings) -> AsyncEngine:
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
+1 -1
View File
@@ -1,4 +1,4 @@
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU."""
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU"""
from pathlib import Path
import resource
+121 -46
View File
@@ -23,12 +23,17 @@ from app.errors.exceptions import (
unavailable,
unsupported_media_type,
)
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error, operation_timeout
from app.errors.handlers import (
RECOVERABLE_STORAGE_ERRORS,
log_recoverable_error,
operation_timeout,
)
from app.schemas.attachment import Attachment
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import MinioStorage
@dataclass(frozen=True)
class CleanupResult:
examined: int
@@ -45,14 +50,14 @@ class OriginalFile:
def require_storage(storage: MinioStorage | None) -> MinioStorage:
if storage is None:
raise unavailable('Объектное хранилище не настроено')
raise unavailable("Объектное хранилище не настроено")
return storage
def validate_declared_size(value: str | None, maximum: int) -> None:
if value is not None:
if not value.isascii() or not value.isdecimal() or len(value) > 20:
raise invalid_request('Некорректный Content-Length')
raise invalid_request("Некорректный Content-Length")
if int(value) > maximum:
raise payload_too_large()
@@ -68,28 +73,45 @@ def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
async def receive_attachment(
session: AsyncSession, storage: MinioStorage | None, user: User, chat_id: UUID,
*, capacity: RequestCapacity, filename: str, content_type: str,
declared_length: str | None, chunks: AsyncIterator[bytes],
session: AsyncSession,
storage: MinioStorage | None,
user: User,
chat_id: UUID,
*,
capacity: RequestCapacity,
filename: str,
content_type: str,
declared_length: str | None,
chunks: AsyncIterator[bytes],
) -> Attachment:
storage = require_storage(storage)
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
timeout_error = invalid_request('Истекло время загрузки')
timeout_error = invalid_request("Истекло время загрузки")
with upload_slot(capacity):
async with operation_timeout(storage.settings.upload_timeout_seconds, timeout_error):
async with operation_timeout(
storage.settings.upload_timeout_seconds, timeout_error
):
result = await upload_attachment(
session, storage, user, chat_id, filename=filename,
content_type=content_type, chunks=chunks,
session,
storage,
user,
chat_id,
filename=filename,
content_type=content_type,
chunks=chunks,
)
return result
async def read_original(
chunks: AsyncIterator[bytes], path: Path, maximum: int, chunk_bytes: int,
chunks: AsyncIterator[bytes],
path: Path,
maximum: int,
chunk_bytes: int,
) -> OriginalFile:
checksum = hashlib.sha256()
size = 0
with path.open('wb') as target:
with path.open("wb") as target:
async for chunk in chunks:
size += len(chunk)
if size > maximum:
@@ -97,13 +119,15 @@ async def read_original(
checksum.update(chunk)
view = memoryview(chunk)
for offset in range(0, len(view), chunk_bytes):
target.write(view[offset:offset + chunk_bytes])
target.write(view[offset : offset + chunk_bytes])
if size == 0:
raise unsupported_media_type('Пустой файл')
raise unsupported_media_type("Пустой файл")
return OriginalFile(path, size, checksum.hexdigest())
async def compensate_upload(session: AsyncSession, storage: MinioStorage, id: UUID) -> None:
async def compensate_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется."""
await session.rollback()
intent = await repository.lock_upload(session, id)
@@ -114,37 +138,54 @@ async def compensate_upload(session: AsyncSession, storage: MinioStorage, id: UU
await session.commit()
async def recover_upload(session: AsyncSession, storage: MinioStorage, id: UUID) -> None:
async def recover_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Локальное восстановление после ошибки; журнал остаётся при отказе компенсации."""
try:
await compensate_upload(session, storage, id)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error('Upload compensation', error)
log_recoverable_error("Upload compensation", error)
# Ошибка повторного rollback не должна скрыть исходную ошибку или отмену.
with suppress(*RECOVERABLE_STORAGE_ERRORS):
await session.rollback()
async def publish_original(
session: AsyncSession, storage: MinioStorage, user: User, chat_id: UUID,
*, original: OriginalFile, filename: str, content_type: str,
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
original: OriginalFile,
filename: str,
content_type: str,
) -> Attachment:
id = uuid4()
key = f'originals/{uuid4().hex}'
key = f"originals/{uuid4().hex}"
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
await repository.reserve_upload(session, id, key)
await session.commit()
try:
if await repository.lock_upload(session, id) is None:
raise unavailable('Загрузка отменена до записи оригинала')
raise unavailable("Загрузка отменена до записи оригинала")
await storage.put_file(
key, original.path, size_bytes=original.size_bytes,
content_type=content_type, sha256=original.sha256,
key,
original.path,
size_bytes=original.size_bytes,
content_type=content_type,
sha256=original.sha256,
)
record = await repository.create_attachment(
session, id=id, chat_id=chat_id, owner_id=user.id, object_key=key,
filename=filename, content_type=content_type,
size_bytes=original.size_bytes, sha256=original.sha256,
session,
id=id,
chat_id=chat_id,
owner_id=user.id,
object_key=key,
filename=filename,
content_type=content_type,
size_bytes=original.size_bytes,
sha256=original.sha256,
)
await repository.finish_upload(session, id)
result = Attachment.model_validate(record)
@@ -158,29 +199,45 @@ async def publish_original(
async def upload_attachment(
session: AsyncSession, storage: MinioStorage, user: User, chat_id: UUID,
*, filename: str, content_type: str, chunks: AsyncIterator[bytes],
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
filename: str,
content_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
await ensure_chat_member(session, user, chat_id)
validate_filename(filename)
if content_type not in {'image/png', 'image/jpeg'}:
raise unsupported_media_type('Принимаются только image/png и image/jpeg')
if content_type not in {"image/png", "image/jpeg"}:
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
# Не держим соединение PostgreSQL во время чтения медленного клиента.
await session.rollback()
settings = storage.settings
with TemporaryDirectory(prefix='messenger-upload-') as directory:
with TemporaryDirectory(prefix="messenger-upload-") as directory:
original = await read_original(
chunks, Path(directory) / 'original', settings.max_upload_bytes, settings.chunk_bytes,
chunks,
Path(directory) / "original",
settings.max_upload_bytes,
settings.chunk_bytes,
)
await validate_image(original.path, content_type, settings)
result = await publish_original(
session, storage, user, chat_id, original=original,
filename=filename, content_type=content_type,
session,
storage,
user,
chat_id,
original=original,
filename=filename,
content_type=content_type,
)
return result
async def get_attachment(session: AsyncSession, user: User, attachment_id: UUID) -> Attachment:
async def get_attachment(
session: AsyncSession, user: User, attachment_id: UUID
) -> Attachment:
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
@@ -189,41 +246,59 @@ async def get_attachment(session: AsyncSession, user: User, attachment_id: UUID)
async def download_attachment(
session: AsyncSession, storage: MinioStorage | None, user: User, attachment_id: UUID,
*, variant: str = 'original',
session: AsyncSession,
storage: MinioStorage | None,
user: User,
attachment_id: UUID,
*,
variant: str = "original",
) -> tuple[Attachment, AsyncIterator[bytes]]:
result = await get_attachment(session, user, attachment_id)
if variant != 'original':
if variant != "original":
raise not_found()
storage = require_storage(storage)
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
key = record.object_key
await session.rollback()
info = await storage.stat_object(key)
if info is None or info.size_bytes != result.size_bytes or info.content_type != result.content_type:
raise unavailable('Оригинал временно недоступен')
if (
info is None
or info.size_bytes != result.size_bytes
or info.content_type != result.content_type
):
raise unavailable("Оригинал временно недоступен")
stream = await storage.open_object(key)
return result, stream
async def validate_message_attachments(
session: AsyncSession, user: User, chat_id: UUID, attachment_ids: list[UUID],
session: AsyncSession,
user: User,
chat_id: UUID,
attachment_ids: list[UUID],
) -> None:
await ensure_chat_member(session, user, chat_id)
records = await repository.get_for_chat(session, chat_id, attachment_ids)
if len(records) != len(attachment_ids):
raise not_found()
if any(record.status != 'ready' for record in records):
raise conflict('Вложение ещё не готово')
if any(record.status != "ready" for record in records):
raise conflict("Вложение ещё не готово")
async def reconcile_attachments(
session: AsyncSession, storage: MinioStorage, *, batch_size: int = 20,
session: AsyncSession,
storage: MinioStorage,
*,
batch_size: int = 20,
dry_run: bool = True,
) -> CleanupResult:
if not 1 <= batch_size <= 100:
raise invalid_request('batch_size должен быть от 1 до 100')
intents = await repository.stale_uploads(session, storage.settings.cleanup_grace_seconds, batch_size)
raise invalid_request("batch_size должен быть от 1 до 100")
intents = await repository.stale_uploads(
session, storage.settings.cleanup_grace_seconds, batch_size
)
candidates = deleted = 0
for intent in intents:
if await repository.get_by_key(session, intent.object_key) is None:
+7 -6
View File
@@ -10,13 +10,14 @@ from app.errors.exceptions import payload_too_large, unsupported_media_type
def decode_image(path: Path, content_type: str, max_pixels: int) -> None:
Image.MAX_IMAGE_PIXELS = max_pixels
warnings.simplefilter('error', Image.DecompressionBombWarning)
with Image.open(path, formats=['PNG', 'JPEG']) as image:
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(path, formats=["PNG", "JPEG"]) as image:
if image.width * image.height > max_pixels:
raise payload_too_large('Превышен лимит пикселей изображения')
if Image.MIME.get(image.format) != content_type:
raise unsupported_media_type('Содержимое не соответствует Content-Type')
raise payload_too_large("Превышен лимит пикселей изображения")
image_format = image.format
if image_format is None or Image.MIME.get(image_format) != content_type:
raise unsupported_media_type("Содержимое не соответствует Content-Type")
image.verify()
# Для JPEG одной verify() недостаточно: декодировать весь оригинал.
with Image.open(path, formats=['PNG', 'JPEG']) as image:
with Image.open(path, formats=["PNG", "JPEG"]) as image:
image.load()
+16 -6
View File
@@ -8,24 +8,34 @@ import unicodedata
from anyio import CancelScope
from app.core.s3 import S3Settings
from app.errors.exceptions import image_validation_error, invalid_request, payload_too_large
from app.errors.exceptions import (
image_validation_error,
invalid_request,
payload_too_large,
)
from app.errors.handlers import operation_timeout
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or any(
char in '/\\' or unicodedata.category(char) in {'Cc', 'Cf', 'Cs'} for char in filename
char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"}
for char in filename
):
raise invalid_request('Недопустимое имя файла')
raise invalid_request("Недопустимое имя файла")
async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None:
process = await asyncio.create_subprocess_exec(
sys.executable, '-m', 'app.core.image_worker', str(path), content_type,
str(settings.max_image_pixels), stdout=asyncio.subprocess.DEVNULL,
sys.executable,
"-m",
"app.core.image_worker",
str(path),
content_type,
str(settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
error = payload_too_large('Превышен бюджет проверки изображения')
error = payload_too_large("Превышен бюджет проверки изображения")
try:
async with operation_timeout(settings.image_timeout_seconds, error):
code = await process.wait()
+9 -3
View File
@@ -13,7 +13,9 @@ class Attachment(Base):
__tablename__ = "attachments"
__table_args__ = (
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"),
CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name="content_type"),
CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
CheckConstraint("status = 'ready'", name="status"),
Index("ix_attachments_chat_id", "chat_id"),
@@ -35,8 +37,12 @@ class Attachment(Base):
class MessageAttachment(Base):
__tablename__ = "message_attachments"
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True)
attachment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("attachments.id"), primary_key=True)
message_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
)
attachment_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("attachments.id"), primary_key=True
)
position: Mapped[int]
+45 -16
View File
@@ -16,7 +16,9 @@ async def create_attachment(session: AsyncSession, **values) -> Attachment:
return record
async def get_attachment(session: AsyncSession, attachment_id: UUID) -> Attachment | None:
async def get_attachment(
session: AsyncSession, attachment_id: UUID
) -> Attachment | None:
return await session.get(Attachment, attachment_id)
@@ -24,27 +26,43 @@ async def get_by_key(session: AsyncSession, key: str) -> Attachment | None:
return await session.scalar(select(Attachment).where(Attachment.object_key == key))
async def get_for_chat(session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]) -> list[Attachment]:
async def get_for_chat(
session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]
) -> list[Attachment]:
records = []
if attachment_ids:
records = list(await session.scalars(select(Attachment).where(
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id,
)))
records = list(
await session.scalars(
select(Attachment).where(
Attachment.id.in_(attachment_ids),
Attachment.chat_id == chat_id,
)
)
)
return records
async def bind_attachments(session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]) -> None:
session.add_all(MessageAttachment(message_id=message_id, attachment_id=id, position=position)
for position, id in enumerate(attachment_ids))
async def bind_attachments(
session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]
) -> None:
session.add_all(
MessageAttachment(message_id=message_id, attachment_id=id, position=position)
for position, id in enumerate(attachment_ids)
)
await session.flush()
async def message_attachment_ids(session: AsyncSession, message_ids: list[UUID]) -> dict[UUID, list[UUID]]:
async def message_attachment_ids(
session: AsyncSession, message_ids: list[UUID]
) -> dict[UUID, list[UUID]]:
result: dict[UUID, list[UUID]] = {id: [] for id in message_ids}
if not message_ids:
return result
rows = await session.execute(select(MessageAttachment).where(
MessageAttachment.message_id.in_(message_ids)).order_by(MessageAttachment.position))
rows = await session.execute(
select(MessageAttachment)
.where(MessageAttachment.message_id.in_(message_ids))
.order_by(MessageAttachment.position)
)
for link in rows.scalars():
result[link.message_id].append(link.attachment_id)
return result
@@ -56,15 +74,26 @@ async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None:
async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None:
return await session.scalar(select(UploadIntent).where(UploadIntent.id == id).with_for_update())
return await session.scalar(
select(UploadIntent).where(UploadIntent.id == id).with_for_update()
)
async def finish_upload(session: AsyncSession, id: UUID) -> None:
await session.execute(delete(UploadIntent).where(UploadIntent.id == id))
async def stale_uploads(session: AsyncSession, grace_seconds: int, limit: int) -> list[UploadIntent]:
result = await session.scalars(select(UploadIntent).where(
UploadIntent.created_at < func.clock_timestamp() - timedelta(seconds=grace_seconds),
).order_by(UploadIntent.created_at).limit(limit).with_for_update(skip_locked=True))
async def stale_uploads(
session: AsyncSession, grace_seconds: int, limit: int
) -> list[UploadIntent]:
result = await session.scalars(
select(UploadIntent)
.where(
UploadIntent.created_at
< func.clock_timestamp() - timedelta(seconds=grace_seconds),
)
.order_by(UploadIntent.created_at)
.limit(limit)
.with_for_update(skip_locked=True)
)
return list(result)
+48 -19
View File
@@ -19,7 +19,9 @@ from app.core.s3 import S3Settings
T = TypeVar("T")
async def run_sync(operation: Callable[..., T], *args, discard_result=None, **kwargs) -> T:
async def run_sync(
operation: Callable[..., T], *args, discard_result=None, **kwargs
) -> T:
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
try:
@@ -31,7 +33,9 @@ async def run_sync(operation: Callable[..., T], *args, discard_result=None, **kw
return result
async def settle_cancelled_call(task: asyncio.Task, discard_result: Callable | None) -> None:
async def settle_cancelled_call(
task: asyncio.Task, discard_result: Callable | None
) -> None:
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
while not task.done():
with suppress(asyncio.CancelledError):
@@ -43,10 +47,12 @@ async def settle_cancelled_call(task: asyncio.Task, discard_result: Callable | N
@dataclass(frozen=True)
class ObjectInfo:
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
key: str
size_bytes: int
content_type: str
last_modified: datetime
size_bytes: int | None
content_type: str | None
last_modified: datetime | None
def close_response(response) -> None:
@@ -74,7 +80,9 @@ class ObjectStream:
if remaining <= 0:
raise TimeoutError
async with asyncio.timeout(remaining):
chunk = await self.storage._call(lambda: self.response.read(self.storage.settings.chunk_bytes))
chunk = await self.storage._call(
lambda: self.response.read(self.storage.settings.chunk_bytes)
)
if not chunk:
await self.aclose()
raise StopAsyncIteration
@@ -95,35 +103,52 @@ class MinioStorage:
# Клиент создаётся в event loop один раз, не конкурентно в worker threads.
url = urlsplit(settings.endpoint)
self._http = urllib3.PoolManager(
timeout=urllib3.Timeout(connect=settings.connect_timeout_seconds, read=settings.read_timeout_seconds),
retries=urllib3.Retry(total=0), maxsize=32,
timeout=urllib3.Timeout(
connect=settings.connect_timeout_seconds,
read=settings.read_timeout_seconds,
),
retries=urllib3.Retry(total=0),
maxsize=32,
)
self._health_http = urllib3.PoolManager(
timeout=urllib3.Timeout(connect=0.25, read=0.5), retries=urllib3.Retry(total=0),
timeout=urllib3.Timeout(connect=0.25, read=0.5),
retries=urllib3.Retry(total=0),
)
self._client = Minio(
url.netloc, secure=url.scheme == "https", region=settings.region,
url.netloc,
secure=url.scheme == "https",
region=settings.region,
access_key=settings.access_key_id_file.read_text().strip(),
secret_key=settings.secret_access_key_file.read_text().strip(), http_client=self._http,
secret_key=settings.secret_access_key_file.read_text().strip(),
http_client=self._http,
)
async def _call(self, operation: Callable, *, discard_result=None):
return await run_sync(operation, discard_result=discard_result)
async def put_file(self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str) -> None:
async def put_file(
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
) -> None:
def put():
with path.open("rb") as source:
# Один PUT: размер части равен максимальному разрешённому файлу.
self._client.put_object(
self.settings.bucket, key, source, size_bytes,
content_type=content_type, metadata={"sha256": sha256},
part_size=10_485_760, num_parallel_uploads=1,
self.settings.bucket,
key,
source,
size_bytes,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10_485_760,
num_parallel_uploads=1,
)
await self._call(put)
async def open_object(self, key: str) -> ObjectStream:
response = await self._call(
lambda: self._client.get_object(self.settings.bucket, key), discard_result=close_response,
lambda: self._client.get_object(self.settings.bucket, key),
discard_result=close_response,
)
return ObjectStream(self, response)
@@ -131,15 +156,19 @@ class MinioStorage:
def stat():
info = self._client.stat_object(self.settings.bucket, key)
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
return await self._call(stat)
async def delete_object(self, key: str) -> None:
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
async def check_available(self) -> bool:
status = await self._call(lambda: self._health_http.request(
"GET", self.settings.endpoint.rstrip("/") + "/minio/health/ready",
).status)
status = await self._call(
lambda: self._health_http.request(
"GET",
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
).status
)
return status == 200
async def aclose(self) -> None: