fix s3 storage
This commit is contained in:
+10
-6
@@ -20,13 +20,13 @@ async def cleanup_once(factory, storage, apply: bool) -> bool:
|
||||
result = await reconcile_attachments(session, storage, dry_run=not apply)
|
||||
print(json.dumps(asdict(result)), flush=True)
|
||||
except RECOVERABLE_STORAGE_ERRORS as error:
|
||||
log_recoverable_error('Reconciliation', error)
|
||||
log_recoverable_error("Reconciliation", error)
|
||||
completed = False
|
||||
return completed
|
||||
|
||||
|
||||
async def run(once: bool, apply: bool) -> None:
|
||||
settings = Settings()
|
||||
settings = Settings() # type: ignore
|
||||
s3_settings = S3Settings()
|
||||
engine = create_db_engine(settings)
|
||||
factory = create_session_factory(engine)
|
||||
@@ -45,12 +45,16 @@ async def run(once: bool, apply: bool) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description='Сверка журнала загрузок MinIO/БД')
|
||||
parser.add_argument('--once', action='store_true')
|
||||
parser.add_argument('--apply', action='store_true', help='удалять безопасные кандидаты; по умолчанию dry-run')
|
||||
parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД")
|
||||
parser.add_argument("--once", action="store_true")
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="удалять безопасные кандидаты; по умолчанию dry-run",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(run(args.once, args.apply))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
+16
-3
@@ -3,13 +3,24 @@ from pathlib import Path
|
||||
from typing import Literal, Self
|
||||
|
||||
from pydantic import Field, SecretStr, model_validator
|
||||
from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
|
||||
from pydantic_settings import (
|
||||
BaseSettings,
|
||||
PydanticBaseSettingsSource,
|
||||
SettingsConfigDict,
|
||||
)
|
||||
from sqlalchemy import URL
|
||||
from app.core.s3 import S3Settings
|
||||
|
||||
ASYNC_DRIVER = "postgresql+asyncpg"
|
||||
SECRETS_DIR = next(
|
||||
(path for path in (Path("/run/secrets"), Path(__file__).resolve().parents[2] / "secrets") if (path / "db_password").is_file()),
|
||||
(
|
||||
path
|
||||
for path in (
|
||||
Path("/run/secrets"),
|
||||
Path(__file__).resolve().parents[2] / "secrets",
|
||||
)
|
||||
if (path / "db_password").is_file()
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -88,7 +99,9 @@ class Settings(BaseSettings):
|
||||
@model_validator(mode="after")
|
||||
def _require_database_source(self) -> Self:
|
||||
if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8:
|
||||
raise ValueError("Drain и graceful shutdown должны суммарно занимать менее 8 секунд")
|
||||
raise ValueError(
|
||||
"Drain и graceful shutdown должны суммарно занимать менее 8 секунд"
|
||||
)
|
||||
if self.database_url is not None:
|
||||
if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER):
|
||||
raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://")
|
||||
|
||||
@@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import (
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
def create_db_engine(settings: Settings) -> AsyncEngine:
|
||||
return create_async_engine(
|
||||
settings.get_database_url(),
|
||||
@@ -25,5 +26,3 @@ def create_db_engine(settings: Settings) -> AsyncEngine:
|
||||
|
||||
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU."""
|
||||
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU"""
|
||||
|
||||
from pathlib import Path
|
||||
import resource
|
||||
|
||||
+121
-46
@@ -23,12 +23,17 @@ from app.errors.exceptions import (
|
||||
unavailable,
|
||||
unsupported_media_type,
|
||||
)
|
||||
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error, operation_timeout
|
||||
from app.errors.handlers import (
|
||||
RECOVERABLE_STORAGE_ERRORS,
|
||||
log_recoverable_error,
|
||||
operation_timeout,
|
||||
)
|
||||
from app.schemas.attachment import Attachment
|
||||
from app.schemas.user import User
|
||||
from app.storage import attachments as repository
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CleanupResult:
|
||||
examined: int
|
||||
@@ -45,14 +50,14 @@ class OriginalFile:
|
||||
|
||||
def require_storage(storage: MinioStorage | None) -> MinioStorage:
|
||||
if storage is None:
|
||||
raise unavailable('Объектное хранилище не настроено')
|
||||
raise unavailable("Объектное хранилище не настроено")
|
||||
return storage
|
||||
|
||||
|
||||
def validate_declared_size(value: str | None, maximum: int) -> None:
|
||||
if value is not None:
|
||||
if not value.isascii() or not value.isdecimal() or len(value) > 20:
|
||||
raise invalid_request('Некорректный Content-Length')
|
||||
raise invalid_request("Некорректный Content-Length")
|
||||
if int(value) > maximum:
|
||||
raise payload_too_large()
|
||||
|
||||
@@ -68,28 +73,45 @@ def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
|
||||
|
||||
|
||||
async def receive_attachment(
|
||||
session: AsyncSession, storage: MinioStorage | None, user: User, chat_id: UUID,
|
||||
*, capacity: RequestCapacity, filename: str, content_type: str,
|
||||
declared_length: str | None, chunks: AsyncIterator[bytes],
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage | None,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
*,
|
||||
capacity: RequestCapacity,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
declared_length: str | None,
|
||||
chunks: AsyncIterator[bytes],
|
||||
) -> Attachment:
|
||||
storage = require_storage(storage)
|
||||
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
|
||||
timeout_error = invalid_request('Истекло время загрузки')
|
||||
timeout_error = invalid_request("Истекло время загрузки")
|
||||
with upload_slot(capacity):
|
||||
async with operation_timeout(storage.settings.upload_timeout_seconds, timeout_error):
|
||||
async with operation_timeout(
|
||||
storage.settings.upload_timeout_seconds, timeout_error
|
||||
):
|
||||
result = await upload_attachment(
|
||||
session, storage, user, chat_id, filename=filename,
|
||||
content_type=content_type, chunks=chunks,
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
chat_id,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
chunks=chunks,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def read_original(
|
||||
chunks: AsyncIterator[bytes], path: Path, maximum: int, chunk_bytes: int,
|
||||
chunks: AsyncIterator[bytes],
|
||||
path: Path,
|
||||
maximum: int,
|
||||
chunk_bytes: int,
|
||||
) -> OriginalFile:
|
||||
checksum = hashlib.sha256()
|
||||
size = 0
|
||||
with path.open('wb') as target:
|
||||
with path.open("wb") as target:
|
||||
async for chunk in chunks:
|
||||
size += len(chunk)
|
||||
if size > maximum:
|
||||
@@ -97,13 +119,15 @@ async def read_original(
|
||||
checksum.update(chunk)
|
||||
view = memoryview(chunk)
|
||||
for offset in range(0, len(view), chunk_bytes):
|
||||
target.write(view[offset:offset + chunk_bytes])
|
||||
target.write(view[offset : offset + chunk_bytes])
|
||||
if size == 0:
|
||||
raise unsupported_media_type('Пустой файл')
|
||||
raise unsupported_media_type("Пустой файл")
|
||||
return OriginalFile(path, size, checksum.hexdigest())
|
||||
|
||||
|
||||
async def compensate_upload(session: AsyncSession, storage: MinioStorage, id: UUID) -> None:
|
||||
async def compensate_upload(
|
||||
session: AsyncSession, storage: MinioStorage, id: UUID
|
||||
) -> None:
|
||||
"""Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется."""
|
||||
await session.rollback()
|
||||
intent = await repository.lock_upload(session, id)
|
||||
@@ -114,37 +138,54 @@ async def compensate_upload(session: AsyncSession, storage: MinioStorage, id: UU
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def recover_upload(session: AsyncSession, storage: MinioStorage, id: UUID) -> None:
|
||||
async def recover_upload(
|
||||
session: AsyncSession, storage: MinioStorage, id: UUID
|
||||
) -> None:
|
||||
"""Локальное восстановление после ошибки; журнал остаётся при отказе компенсации."""
|
||||
try:
|
||||
await compensate_upload(session, storage, id)
|
||||
except RECOVERABLE_STORAGE_ERRORS as error:
|
||||
log_recoverable_error('Upload compensation', error)
|
||||
log_recoverable_error("Upload compensation", error)
|
||||
# Ошибка повторного rollback не должна скрыть исходную ошибку или отмену.
|
||||
with suppress(*RECOVERABLE_STORAGE_ERRORS):
|
||||
await session.rollback()
|
||||
|
||||
|
||||
async def publish_original(
|
||||
session: AsyncSession, storage: MinioStorage, user: User, chat_id: UUID,
|
||||
*, original: OriginalFile, filename: str, content_type: str,
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
*,
|
||||
original: OriginalFile,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
) -> Attachment:
|
||||
id = uuid4()
|
||||
key = f'originals/{uuid4().hex}'
|
||||
key = f"originals/{uuid4().hex}"
|
||||
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
|
||||
await repository.reserve_upload(session, id, key)
|
||||
await session.commit()
|
||||
try:
|
||||
if await repository.lock_upload(session, id) is None:
|
||||
raise unavailable('Загрузка отменена до записи оригинала')
|
||||
raise unavailable("Загрузка отменена до записи оригинала")
|
||||
await storage.put_file(
|
||||
key, original.path, size_bytes=original.size_bytes,
|
||||
content_type=content_type, sha256=original.sha256,
|
||||
key,
|
||||
original.path,
|
||||
size_bytes=original.size_bytes,
|
||||
content_type=content_type,
|
||||
sha256=original.sha256,
|
||||
)
|
||||
record = await repository.create_attachment(
|
||||
session, id=id, chat_id=chat_id, owner_id=user.id, object_key=key,
|
||||
filename=filename, content_type=content_type,
|
||||
size_bytes=original.size_bytes, sha256=original.sha256,
|
||||
session,
|
||||
id=id,
|
||||
chat_id=chat_id,
|
||||
owner_id=user.id,
|
||||
object_key=key,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=original.size_bytes,
|
||||
sha256=original.sha256,
|
||||
)
|
||||
await repository.finish_upload(session, id)
|
||||
result = Attachment.model_validate(record)
|
||||
@@ -158,29 +199,45 @@ async def publish_original(
|
||||
|
||||
|
||||
async def upload_attachment(
|
||||
session: AsyncSession, storage: MinioStorage, user: User, chat_id: UUID,
|
||||
*, filename: str, content_type: str, chunks: AsyncIterator[bytes],
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
*,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
chunks: AsyncIterator[bytes],
|
||||
) -> Attachment:
|
||||
await ensure_chat_member(session, user, chat_id)
|
||||
validate_filename(filename)
|
||||
if content_type not in {'image/png', 'image/jpeg'}:
|
||||
raise unsupported_media_type('Принимаются только image/png и image/jpeg')
|
||||
if content_type not in {"image/png", "image/jpeg"}:
|
||||
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
|
||||
# Не держим соединение PostgreSQL во время чтения медленного клиента.
|
||||
await session.rollback()
|
||||
settings = storage.settings
|
||||
with TemporaryDirectory(prefix='messenger-upload-') as directory:
|
||||
with TemporaryDirectory(prefix="messenger-upload-") as directory:
|
||||
original = await read_original(
|
||||
chunks, Path(directory) / 'original', settings.max_upload_bytes, settings.chunk_bytes,
|
||||
chunks,
|
||||
Path(directory) / "original",
|
||||
settings.max_upload_bytes,
|
||||
settings.chunk_bytes,
|
||||
)
|
||||
await validate_image(original.path, content_type, settings)
|
||||
result = await publish_original(
|
||||
session, storage, user, chat_id, original=original,
|
||||
filename=filename, content_type=content_type,
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
chat_id,
|
||||
original=original,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def get_attachment(session: AsyncSession, user: User, attachment_id: UUID) -> Attachment:
|
||||
async def get_attachment(
|
||||
session: AsyncSession, user: User, attachment_id: UUID
|
||||
) -> Attachment:
|
||||
record = await repository.get_attachment(session, attachment_id)
|
||||
if record is None:
|
||||
raise not_found()
|
||||
@@ -189,41 +246,59 @@ async def get_attachment(session: AsyncSession, user: User, attachment_id: UUID)
|
||||
|
||||
|
||||
async def download_attachment(
|
||||
session: AsyncSession, storage: MinioStorage | None, user: User, attachment_id: UUID,
|
||||
*, variant: str = 'original',
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage | None,
|
||||
user: User,
|
||||
attachment_id: UUID,
|
||||
*,
|
||||
variant: str = "original",
|
||||
) -> tuple[Attachment, AsyncIterator[bytes]]:
|
||||
result = await get_attachment(session, user, attachment_id)
|
||||
if variant != 'original':
|
||||
if variant != "original":
|
||||
raise not_found()
|
||||
storage = require_storage(storage)
|
||||
record = await repository.get_attachment(session, attachment_id)
|
||||
if record is None:
|
||||
raise not_found()
|
||||
key = record.object_key
|
||||
await session.rollback()
|
||||
info = await storage.stat_object(key)
|
||||
if info is None or info.size_bytes != result.size_bytes or info.content_type != result.content_type:
|
||||
raise unavailable('Оригинал временно недоступен')
|
||||
if (
|
||||
info is None
|
||||
or info.size_bytes != result.size_bytes
|
||||
or info.content_type != result.content_type
|
||||
):
|
||||
raise unavailable("Оригинал временно недоступен")
|
||||
stream = await storage.open_object(key)
|
||||
return result, stream
|
||||
|
||||
|
||||
async def validate_message_attachments(
|
||||
session: AsyncSession, user: User, chat_id: UUID, attachment_ids: list[UUID],
|
||||
session: AsyncSession,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
attachment_ids: list[UUID],
|
||||
) -> None:
|
||||
await ensure_chat_member(session, user, chat_id)
|
||||
records = await repository.get_for_chat(session, chat_id, attachment_ids)
|
||||
if len(records) != len(attachment_ids):
|
||||
raise not_found()
|
||||
if any(record.status != 'ready' for record in records):
|
||||
raise conflict('Вложение ещё не готово')
|
||||
if any(record.status != "ready" for record in records):
|
||||
raise conflict("Вложение ещё не готово")
|
||||
|
||||
|
||||
async def reconcile_attachments(
|
||||
session: AsyncSession, storage: MinioStorage, *, batch_size: int = 20,
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
*,
|
||||
batch_size: int = 20,
|
||||
dry_run: bool = True,
|
||||
) -> CleanupResult:
|
||||
if not 1 <= batch_size <= 100:
|
||||
raise invalid_request('batch_size должен быть от 1 до 100')
|
||||
intents = await repository.stale_uploads(session, storage.settings.cleanup_grace_seconds, batch_size)
|
||||
raise invalid_request("batch_size должен быть от 1 до 100")
|
||||
intents = await repository.stale_uploads(
|
||||
session, storage.settings.cleanup_grace_seconds, batch_size
|
||||
)
|
||||
candidates = deleted = 0
|
||||
for intent in intents:
|
||||
if await repository.get_by_key(session, intent.object_key) is None:
|
||||
|
||||
@@ -10,13 +10,14 @@ from app.errors.exceptions import payload_too_large, unsupported_media_type
|
||||
|
||||
def decode_image(path: Path, content_type: str, max_pixels: int) -> None:
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
with Image.open(path, formats=['PNG', 'JPEG']) as image:
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(path, formats=["PNG", "JPEG"]) as image:
|
||||
if image.width * image.height > max_pixels:
|
||||
raise payload_too_large('Превышен лимит пикселей изображения')
|
||||
if Image.MIME.get(image.format) != content_type:
|
||||
raise unsupported_media_type('Содержимое не соответствует Content-Type')
|
||||
raise payload_too_large("Превышен лимит пикселей изображения")
|
||||
image_format = image.format
|
||||
if image_format is None or Image.MIME.get(image_format) != content_type:
|
||||
raise unsupported_media_type("Содержимое не соответствует Content-Type")
|
||||
image.verify()
|
||||
# Для JPEG одной verify() недостаточно: декодировать весь оригинал.
|
||||
with Image.open(path, formats=['PNG', 'JPEG']) as image:
|
||||
with Image.open(path, formats=["PNG", "JPEG"]) as image:
|
||||
image.load()
|
||||
|
||||
+16
-6
@@ -8,24 +8,34 @@ import unicodedata
|
||||
from anyio import CancelScope
|
||||
|
||||
from app.core.s3 import S3Settings
|
||||
from app.errors.exceptions import image_validation_error, invalid_request, payload_too_large
|
||||
from app.errors.exceptions import (
|
||||
image_validation_error,
|
||||
invalid_request,
|
||||
payload_too_large,
|
||||
)
|
||||
from app.errors.handlers import operation_timeout
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> None:
|
||||
if not 1 <= len(filename) <= 255 or any(
|
||||
char in '/\\' or unicodedata.category(char) in {'Cc', 'Cf', 'Cs'} for char in filename
|
||||
char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"}
|
||||
for char in filename
|
||||
):
|
||||
raise invalid_request('Недопустимое имя файла')
|
||||
raise invalid_request("Недопустимое имя файла")
|
||||
|
||||
|
||||
async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, '-m', 'app.core.image_worker', str(path), content_type,
|
||||
str(settings.max_image_pixels), stdout=asyncio.subprocess.DEVNULL,
|
||||
sys.executable,
|
||||
"-m",
|
||||
"app.core.image_worker",
|
||||
str(path),
|
||||
content_type,
|
||||
str(settings.max_image_pixels),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
error = payload_too_large('Превышен бюджет проверки изображения')
|
||||
error = payload_too_large("Превышен бюджет проверки изображения")
|
||||
try:
|
||||
async with operation_timeout(settings.image_timeout_seconds, error):
|
||||
code = await process.wait()
|
||||
|
||||
@@ -13,7 +13,9 @@ class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
__table_args__ = (
|
||||
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"),
|
||||
CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name="content_type"),
|
||||
CheckConstraint(
|
||||
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
|
||||
),
|
||||
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
|
||||
CheckConstraint("status = 'ready'", name="status"),
|
||||
Index("ix_attachments_chat_id", "chat_id"),
|
||||
@@ -35,8 +37,12 @@ class Attachment(Base):
|
||||
class MessageAttachment(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("attachments.id"), primary_key=True)
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("attachments.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int]
|
||||
|
||||
|
||||
|
||||
+45
-16
@@ -16,7 +16,9 @@ async def create_attachment(session: AsyncSession, **values) -> Attachment:
|
||||
return record
|
||||
|
||||
|
||||
async def get_attachment(session: AsyncSession, attachment_id: UUID) -> Attachment | None:
|
||||
async def get_attachment(
|
||||
session: AsyncSession, attachment_id: UUID
|
||||
) -> Attachment | None:
|
||||
return await session.get(Attachment, attachment_id)
|
||||
|
||||
|
||||
@@ -24,27 +26,43 @@ async def get_by_key(session: AsyncSession, key: str) -> Attachment | None:
|
||||
return await session.scalar(select(Attachment).where(Attachment.object_key == key))
|
||||
|
||||
|
||||
async def get_for_chat(session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]) -> list[Attachment]:
|
||||
async def get_for_chat(
|
||||
session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]
|
||||
) -> list[Attachment]:
|
||||
records = []
|
||||
if attachment_ids:
|
||||
records = list(await session.scalars(select(Attachment).where(
|
||||
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id,
|
||||
)))
|
||||
records = list(
|
||||
await session.scalars(
|
||||
select(Attachment).where(
|
||||
Attachment.id.in_(attachment_ids),
|
||||
Attachment.chat_id == chat_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
async def bind_attachments(session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]) -> None:
|
||||
session.add_all(MessageAttachment(message_id=message_id, attachment_id=id, position=position)
|
||||
for position, id in enumerate(attachment_ids))
|
||||
async def bind_attachments(
|
||||
session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]
|
||||
) -> None:
|
||||
session.add_all(
|
||||
MessageAttachment(message_id=message_id, attachment_id=id, position=position)
|
||||
for position, id in enumerate(attachment_ids)
|
||||
)
|
||||
await session.flush()
|
||||
|
||||
|
||||
async def message_attachment_ids(session: AsyncSession, message_ids: list[UUID]) -> dict[UUID, list[UUID]]:
|
||||
async def message_attachment_ids(
|
||||
session: AsyncSession, message_ids: list[UUID]
|
||||
) -> dict[UUID, list[UUID]]:
|
||||
result: dict[UUID, list[UUID]] = {id: [] for id in message_ids}
|
||||
if not message_ids:
|
||||
return result
|
||||
rows = await session.execute(select(MessageAttachment).where(
|
||||
MessageAttachment.message_id.in_(message_ids)).order_by(MessageAttachment.position))
|
||||
rows = await session.execute(
|
||||
select(MessageAttachment)
|
||||
.where(MessageAttachment.message_id.in_(message_ids))
|
||||
.order_by(MessageAttachment.position)
|
||||
)
|
||||
for link in rows.scalars():
|
||||
result[link.message_id].append(link.attachment_id)
|
||||
return result
|
||||
@@ -56,15 +74,26 @@ async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None:
|
||||
|
||||
|
||||
async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None:
|
||||
return await session.scalar(select(UploadIntent).where(UploadIntent.id == id).with_for_update())
|
||||
return await session.scalar(
|
||||
select(UploadIntent).where(UploadIntent.id == id).with_for_update()
|
||||
)
|
||||
|
||||
|
||||
async def finish_upload(session: AsyncSession, id: UUID) -> None:
|
||||
await session.execute(delete(UploadIntent).where(UploadIntent.id == id))
|
||||
|
||||
|
||||
async def stale_uploads(session: AsyncSession, grace_seconds: int, limit: int) -> list[UploadIntent]:
|
||||
result = await session.scalars(select(UploadIntent).where(
|
||||
UploadIntent.created_at < func.clock_timestamp() - timedelta(seconds=grace_seconds),
|
||||
).order_by(UploadIntent.created_at).limit(limit).with_for_update(skip_locked=True))
|
||||
async def stale_uploads(
|
||||
session: AsyncSession, grace_seconds: int, limit: int
|
||||
) -> list[UploadIntent]:
|
||||
result = await session.scalars(
|
||||
select(UploadIntent)
|
||||
.where(
|
||||
UploadIntent.created_at
|
||||
< func.clock_timestamp() - timedelta(seconds=grace_seconds),
|
||||
)
|
||||
.order_by(UploadIntent.created_at)
|
||||
.limit(limit)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
return list(result)
|
||||
|
||||
+48
-19
@@ -19,7 +19,9 @@ from app.core.s3 import S3Settings
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
async def run_sync(operation: Callable[..., T], *args, discard_result=None, **kwargs) -> T:
|
||||
async def run_sync(
|
||||
operation: Callable[..., T], *args, discard_result=None, **kwargs
|
||||
) -> T:
|
||||
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
|
||||
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
|
||||
try:
|
||||
@@ -31,7 +33,9 @@ async def run_sync(operation: Callable[..., T], *args, discard_result=None, **kw
|
||||
return result
|
||||
|
||||
|
||||
async def settle_cancelled_call(task: asyncio.Task, discard_result: Callable | None) -> None:
|
||||
async def settle_cancelled_call(
|
||||
task: asyncio.Task, discard_result: Callable | None
|
||||
) -> None:
|
||||
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
|
||||
while not task.done():
|
||||
with suppress(asyncio.CancelledError):
|
||||
@@ -43,10 +47,12 @@ async def settle_cancelled_call(task: asyncio.Task, discard_result: Callable | N
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjectInfo:
|
||||
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
|
||||
|
||||
key: str
|
||||
size_bytes: int
|
||||
content_type: str
|
||||
last_modified: datetime
|
||||
size_bytes: int | None
|
||||
content_type: str | None
|
||||
last_modified: datetime | None
|
||||
|
||||
|
||||
def close_response(response) -> None:
|
||||
@@ -74,7 +80,9 @@ class ObjectStream:
|
||||
if remaining <= 0:
|
||||
raise TimeoutError
|
||||
async with asyncio.timeout(remaining):
|
||||
chunk = await self.storage._call(lambda: self.response.read(self.storage.settings.chunk_bytes))
|
||||
chunk = await self.storage._call(
|
||||
lambda: self.response.read(self.storage.settings.chunk_bytes)
|
||||
)
|
||||
if not chunk:
|
||||
await self.aclose()
|
||||
raise StopAsyncIteration
|
||||
@@ -95,35 +103,52 @@ class MinioStorage:
|
||||
# Клиент создаётся в event loop один раз, не конкурентно в worker threads.
|
||||
url = urlsplit(settings.endpoint)
|
||||
self._http = urllib3.PoolManager(
|
||||
timeout=urllib3.Timeout(connect=settings.connect_timeout_seconds, read=settings.read_timeout_seconds),
|
||||
retries=urllib3.Retry(total=0), maxsize=32,
|
||||
timeout=urllib3.Timeout(
|
||||
connect=settings.connect_timeout_seconds,
|
||||
read=settings.read_timeout_seconds,
|
||||
),
|
||||
retries=urllib3.Retry(total=0),
|
||||
maxsize=32,
|
||||
)
|
||||
self._health_http = urllib3.PoolManager(
|
||||
timeout=urllib3.Timeout(connect=0.25, read=0.5), retries=urllib3.Retry(total=0),
|
||||
timeout=urllib3.Timeout(connect=0.25, read=0.5),
|
||||
retries=urllib3.Retry(total=0),
|
||||
)
|
||||
self._client = Minio(
|
||||
url.netloc, secure=url.scheme == "https", region=settings.region,
|
||||
url.netloc,
|
||||
secure=url.scheme == "https",
|
||||
region=settings.region,
|
||||
access_key=settings.access_key_id_file.read_text().strip(),
|
||||
secret_key=settings.secret_access_key_file.read_text().strip(), http_client=self._http,
|
||||
secret_key=settings.secret_access_key_file.read_text().strip(),
|
||||
http_client=self._http,
|
||||
)
|
||||
|
||||
async def _call(self, operation: Callable, *, discard_result=None):
|
||||
return await run_sync(operation, discard_result=discard_result)
|
||||
|
||||
async def put_file(self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str) -> None:
|
||||
async def put_file(
|
||||
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
|
||||
) -> None:
|
||||
def put():
|
||||
with path.open("rb") as source:
|
||||
# Один PUT: размер части равен максимальному разрешённому файлу.
|
||||
self._client.put_object(
|
||||
self.settings.bucket, key, source, size_bytes,
|
||||
content_type=content_type, metadata={"sha256": sha256},
|
||||
part_size=10_485_760, num_parallel_uploads=1,
|
||||
self.settings.bucket,
|
||||
key,
|
||||
source,
|
||||
size_bytes,
|
||||
content_type=content_type,
|
||||
metadata={"sha256": sha256},
|
||||
part_size=10_485_760,
|
||||
num_parallel_uploads=1,
|
||||
)
|
||||
|
||||
await self._call(put)
|
||||
|
||||
async def open_object(self, key: str) -> ObjectStream:
|
||||
response = await self._call(
|
||||
lambda: self._client.get_object(self.settings.bucket, key), discard_result=close_response,
|
||||
lambda: self._client.get_object(self.settings.bucket, key),
|
||||
discard_result=close_response,
|
||||
)
|
||||
return ObjectStream(self, response)
|
||||
|
||||
@@ -131,15 +156,19 @@ class MinioStorage:
|
||||
def stat():
|
||||
info = self._client.stat_object(self.settings.bucket, key)
|
||||
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
|
||||
|
||||
return await self._call(stat)
|
||||
|
||||
async def delete_object(self, key: str) -> None:
|
||||
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
|
||||
|
||||
async def check_available(self) -> bool:
|
||||
status = await self._call(lambda: self._health_http.request(
|
||||
"GET", self.settings.endpoint.rstrip("/") + "/minio/health/ready",
|
||||
).status)
|
||||
status = await self._call(
|
||||
lambda: self._health_http.request(
|
||||
"GET",
|
||||
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
|
||||
).status
|
||||
)
|
||||
return status == 200
|
||||
|
||||
async def aclose(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user