220 lines
8.1 KiB
Python
220 lines
8.1 KiB
Python
"""
|
|
In-memory реализации интерфейсов с внедрением отказов.
|
|
|
|
fail — имена методов, которые бросают UnavailableError (как репозитории
|
|
при недоступной БД/S3); hang — методы, которые зависают.
|
|
"""
|
|
|
|
import asyncio
|
|
import io
|
|
from collections.abc import AsyncIterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import BinaryIO
|
|
from uuid import UUID, uuid4
|
|
|
|
from domain import (
|
|
AttachmentSchema,
|
|
AttachmentStatus,
|
|
ChatSchema,
|
|
NewAttachmentSchema,
|
|
PageKey,
|
|
StoredAttachmentSchema,
|
|
UnfinishedUploadSchema,
|
|
)
|
|
from domain.errors import UnavailableError
|
|
from interfaces import AttachmentRepository, ChatRepository, ObjectStorage, StoredObject
|
|
|
|
HANG_SEC = 3600
|
|
|
|
|
|
class FaultInjection:
|
|
def __init__(self) -> None:
|
|
self.fail: set[str] = set()
|
|
self.hang: set[str] = set()
|
|
self.calls: list[str] = []
|
|
|
|
async def _enter(self, name: str) -> None:
|
|
self.calls.append(name)
|
|
if name in self.hang:
|
|
await asyncio.sleep(HANG_SEC)
|
|
if name in self.fail:
|
|
raise UnavailableError()
|
|
|
|
|
|
class FakeChatRepository(ChatRepository):
|
|
"""Пользователь — участник любого чата, кроме перечисленных в foreign"""
|
|
|
|
def __init__(self) -> None:
|
|
self.foreign: set[UUID] = set()
|
|
|
|
async def create(self, title: str, created_by: UUID, member_ids: list[UUID]) -> ChatSchema:
|
|
raise NotImplementedError
|
|
|
|
async def get_for_member(self, chat_id: UUID, member_id: UUID) -> ChatSchema | None:
|
|
if chat_id in self.foreign:
|
|
return None
|
|
return ChatSchema(
|
|
id=chat_id, title="chat", member_ids=[member_id], created_at=datetime.now(UTC)
|
|
)
|
|
|
|
async def list_for_member(
|
|
self, member_id: UUID, after: PageKey | None, limit: int
|
|
) -> list[ChatSchema]:
|
|
raise NotImplementedError
|
|
|
|
|
|
class FakeAttachmentRepository(FaultInjection, AttachmentRepository):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.rows: dict[UUID, dict] = {}
|
|
self.ready_returns_none = False
|
|
|
|
def add_row(self, object_key: str, status: AttachmentStatus, age_seconds: float) -> UUID:
|
|
attachment_id = uuid4()
|
|
self.rows[attachment_id] = {
|
|
"status": status,
|
|
"object_key": object_key,
|
|
"created_at": datetime.now(UTC) - timedelta(seconds=age_seconds),
|
|
"new": None,
|
|
}
|
|
return attachment_id
|
|
|
|
async def create_pending(self, attachment: NewAttachmentSchema) -> UUID:
|
|
await self._enter("create_pending")
|
|
attachment_id = self.add_row(attachment.object_key, AttachmentStatus.PENDING, 0)
|
|
self.rows[attachment_id]["new"] = attachment
|
|
return attachment_id
|
|
|
|
async def mark_ready(self, attachment_id: UUID, grace_seconds: int) -> AttachmentSchema | None:
|
|
await self._enter("mark_ready")
|
|
row = self.rows.get(attachment_id)
|
|
if self.ready_returns_none or row is None or row["status"] != AttachmentStatus.PENDING:
|
|
return None
|
|
if row["created_at"] <= datetime.now(UTC) - timedelta(seconds=grace_seconds):
|
|
return None
|
|
row["status"] = AttachmentStatus.READY
|
|
new: NewAttachmentSchema = row["new"]
|
|
return AttachmentSchema(
|
|
id=attachment_id,
|
|
chat_id=new.chat_id,
|
|
owner_id=new.owner_id,
|
|
filename=new.filename,
|
|
content_type=new.content_type,
|
|
size_bytes=new.size_bytes,
|
|
sha256=new.sha256.hex(),
|
|
created_at=row["created_at"],
|
|
updated_at=datetime.now(UTC),
|
|
)
|
|
|
|
async def claim_pending(self, attachment_id: UUID) -> UnfinishedUploadSchema | None:
|
|
await self._enter("claim_pending")
|
|
row = self.rows.get(attachment_id)
|
|
if row is None or row["status"] != AttachmentStatus.PENDING:
|
|
return None
|
|
row["status"] = AttachmentStatus.DELETING
|
|
return UnfinishedUploadSchema(id=attachment_id, object_key=row["object_key"])
|
|
|
|
def _unfinished(self, grace_seconds: int) -> list[UUID]:
|
|
threshold = datetime.now(UTC) - timedelta(seconds=grace_seconds)
|
|
return [
|
|
attachment_id
|
|
for attachment_id, row in self.rows.items()
|
|
if row["status"] == AttachmentStatus.DELETING
|
|
or (row["status"] == AttachmentStatus.PENDING and row["created_at"] < threshold)
|
|
]
|
|
|
|
async def claim_unfinished(self, grace_seconds: int, limit: int) -> list[UnfinishedUploadSchema]:
|
|
await self._enter("claim_unfinished")
|
|
# Повторный вызов в том же проходе не должен возвращать те же строки
|
|
claimed = [
|
|
attachment_id
|
|
for attachment_id in self._unfinished(grace_seconds)
|
|
if not self.rows[attachment_id].get("claimed_now")
|
|
][:limit]
|
|
for attachment_id in claimed:
|
|
self.rows[attachment_id]["status"] = AttachmentStatus.DELETING
|
|
self.rows[attachment_id]["claimed_now"] = True
|
|
return [
|
|
UnfinishedUploadSchema(id=i, object_key=self.rows[i]["object_key"]) for i in claimed
|
|
]
|
|
|
|
async def list_unfinished(self, grace_seconds: int, limit: int) -> list[UnfinishedUploadSchema]:
|
|
await self._enter("list_unfinished")
|
|
return [
|
|
UnfinishedUploadSchema(id=i, object_key=self.rows[i]["object_key"])
|
|
for i in self._unfinished(grace_seconds)[:limit]
|
|
]
|
|
|
|
async def delete_claimed(self, attachment_ids: list[UUID]) -> int:
|
|
await self._enter("delete_claimed")
|
|
deleted = [
|
|
i
|
|
for i in attachment_ids
|
|
if i in self.rows and self.rows[i]["status"] == AttachmentStatus.DELETING
|
|
]
|
|
for attachment_id in deleted:
|
|
del self.rows[attachment_id]
|
|
return len(deleted)
|
|
|
|
async def get_for_member(
|
|
self, attachment_id: UUID, member_id: UUID
|
|
) -> StoredAttachmentSchema | None:
|
|
raise NotImplementedError
|
|
|
|
async def find_known_keys(self, object_keys: list[str]) -> set[str]:
|
|
await self._enter("find_known_keys")
|
|
known = {row["object_key"] for row in self.rows.values()}
|
|
return {key for key in object_keys if key in known}
|
|
|
|
def statuses(self) -> list[str]:
|
|
return sorted(row["status"] for row in self.rows.values())
|
|
|
|
|
|
class FakeStoredObject(StoredObject):
|
|
def __init__(self, data: bytes) -> None:
|
|
self._data = data
|
|
self.content_length = len(data)
|
|
|
|
async def iter_chunks(self) -> AsyncIterator[bytes]:
|
|
yield self._data
|
|
|
|
async def close(self) -> None:
|
|
pass
|
|
|
|
|
|
class FakeObjectStorage(FaultInjection, ObjectStorage):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.objects: dict[str, tuple[bytes, datetime]] = {}
|
|
|
|
def add_object(self, key: str, age_seconds: float) -> None:
|
|
self.objects[key] = (b"x", datetime.now(UTC) - timedelta(seconds=age_seconds))
|
|
|
|
async def put(
|
|
self, key: str, body: BinaryIO, size: int, content_type: str, sha256: bytes
|
|
) -> None:
|
|
# Как S3: объект появляется до того, как вызывающий получит ответ
|
|
self.objects[key] = (body.read(), datetime.now(UTC))
|
|
await self._enter("put")
|
|
|
|
async def open(self, key: str) -> StoredObject | None:
|
|
await self._enter("open")
|
|
stored = self.objects.get(key)
|
|
return FakeStoredObject(stored[0]) if stored else None
|
|
|
|
async def delete(self, key: str) -> None:
|
|
await self._enter("delete")
|
|
self.objects.pop(key, None)
|
|
|
|
async def list_objects(self, prefix: str) -> AsyncIterator[tuple[str, datetime]]:
|
|
await self._enter("list_objects")
|
|
for key, (_, modified_at) in list(self.objects.items()):
|
|
if key.startswith(prefix):
|
|
yield key, modified_at
|
|
|
|
|
|
async def chunks_of(data: bytes, size: int = 64 * 1024) -> AsyncIterator[bytes]:
|
|
stream = io.BytesIO(data)
|
|
while chunk := stream.read(size):
|
|
yield chunk
|