StorageReconciler walks the sorted bucket listing and keyset pages of attachments ordered by object key side by side, so a pass is linear and keeps one page of each side in memory. An object without a row is deleted only when it is older than grace plus skew and a lookup by key right before the delete still finds no row; a ready row whose object is missing, has another size or, with the deep check, another digest is put in quarantine after a HEAD recheck and is never deleted; a stale intent is claimed with SKIP LOCKED and dropped object first; a quarantined row whose object matches again is released. Deep checks hash objects in a bounded task group, each action uses its own unit of work, and failures of single actions are counted instead of aborting. An advisory lock lets one pass run at a time. Content of a quarantined attachment answers 404 without touching S3, its metadata is unchanged. The storage port gains listing pages and HEAD, the repository keyset pages, lookup by key, claim and quarantine marking; the platform logger can write to another stream. The one-off entry point python -m messenger.attachments.entrypoints.reconcile is a dry run unless --apply is given, assembles the adapters with the API key of S3 and prints a JSON report with counts per class, samples and the cost of the pass to stdout while the journal goes to stderr. Tests run the reconciler against its own migrated database with every class, apply and repeat, the row that appears after the listing, the busy lock and the release, and check paged byte-order listing and HEAD against RustFS. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
88 lines
3.0 KiB
Python
88 lines
3.0 KiB
Python
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
|
|
from dishka import Provider, Scope, provide
|
|
|
|
from messenger.attachments.application.errors import ObjectMissingError
|
|
from messenger.attachments.application.ports import ObjectContent, ObjectStorage, StoredObject
|
|
from messenger.shared.application.errors import Unavailable
|
|
|
|
|
|
class NullLogger:
|
|
__slots__ = ()
|
|
|
|
def info(self, event: str, /, **fields: Any) -> None: ...
|
|
|
|
def warning(self, event: str, /, **fields: Any) -> None: ...
|
|
|
|
def error(self, event: str, /, **fields: Any) -> None: ...
|
|
|
|
|
|
class FrozenClock:
|
|
__slots__ = ("_current",)
|
|
|
|
def __init__(self, start: datetime) -> None:
|
|
self._current = start
|
|
|
|
def now(self) -> datetime:
|
|
self._current += timedelta(microseconds=1)
|
|
return self._current
|
|
|
|
|
|
async def pieces(data: bytes, size: int = 65_536) -> AsyncIterator[bytes]:
|
|
for start in range(0, len(data), size):
|
|
yield data[start : start + size]
|
|
|
|
|
|
class MemoryObjectStorage:
|
|
__slots__ = ("after_page", "after_put", "failing", "modified", "objects")
|
|
|
|
def __init__(self) -> None:
|
|
self.objects: dict[str, tuple[bytes, str]] = {}
|
|
self.modified: dict[str, datetime] = {}
|
|
self.failing = False
|
|
self.after_put: Callable[[str], Awaitable[None]] | None = None
|
|
self.after_page: Callable[[], Awaitable[None]] | None = None
|
|
|
|
async def put(self, key: str, data: bytes, content_type: str) -> None:
|
|
if self.failing:
|
|
raise Unavailable("S3 недоступно")
|
|
self.objects[key] = (bytes(data), content_type)
|
|
self.modified[key] = datetime.now(UTC)
|
|
if self.after_put is not None:
|
|
await self.after_put(key)
|
|
|
|
async def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]:
|
|
if self.failing:
|
|
raise Unavailable("S3 недоступно")
|
|
keys = sorted(key for key in self.objects if key.startswith(prefix))
|
|
for start in range(0, len(keys), page_size):
|
|
yield [self._stored(key) for key in keys[start : start + page_size]]
|
|
if self.after_page is not None:
|
|
await self.after_page()
|
|
|
|
async def stat(self, key: str) -> StoredObject | None:
|
|
return self._stored(key) if key in self.objects else None
|
|
|
|
def _stored(self, key: str) -> StoredObject:
|
|
return StoredObject(key, len(self.objects[key][0]), self.modified[key])
|
|
|
|
async def open(self, key: str) -> ObjectContent:
|
|
if self.failing:
|
|
raise Unavailable("S3 недоступно")
|
|
if key not in self.objects:
|
|
raise ObjectMissingError(key)
|
|
data = self.objects[key][0]
|
|
return ObjectContent(size=len(data), chunks=pieces(data))
|
|
|
|
async def delete(self, key: str) -> None:
|
|
self.objects.pop(key, None)
|
|
self.modified.pop(key, None)
|
|
|
|
|
|
class MemoryStorageProvider(Provider):
|
|
@provide(scope=Scope.APP)
|
|
def storage(self) -> ObjectStorage:
|
|
return MemoryObjectStorage()
|