Files
mtusi2026lab2/student-tests/fakes.py
T
ArtyomandClaude Opus 5.5 91731fd317 Reconcile attachment metadata with the bucket in one merge pass
StorageReconciler walks the sorted bucket listing and keyset pages of
attachments ordered by object key side by side, so a pass is linear
and keeps one page of each side in memory. An object without a row is
deleted only when it is older than grace plus skew and a lookup by key
right before the delete still finds no row; a ready row whose object
is missing, has another size or, with the deep check, another digest
is put in quarantine after a HEAD recheck and is never deleted; a stale
intent is claimed with SKIP LOCKED and dropped object first; a
quarantined row whose object matches again is released. Deep checks
hash objects in a bounded task group, each action uses its own unit of
work, and failures of single actions are counted instead of aborting.
An advisory lock lets one pass run at a time. Content of a quarantined
attachment answers 404 without touching S3, its metadata is unchanged.

The storage port gains listing pages and HEAD, the repository keyset
pages, lookup by key, claim and quarantine marking; the platform logger
can write to another stream. The one-off entry point
python -m messenger.attachments.entrypoints.reconcile is a dry run
unless --apply is given, assembles the adapters with the API key of S3
and prints a JSON report with counts per class, samples and the cost
of the pass to stdout while the journal goes to stderr.

Tests run the reconciler against its own migrated database with every
class, apply and repeat, the row that appears after the listing, the
busy lock and the release, and check paged byte-order listing and HEAD
against RustFS.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-29 03:12:10 +03:00

88 lines
3.0 KiB
Python

from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
from datetime import UTC, datetime, timedelta
from typing import Any
from dishka import Provider, Scope, provide
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.application.ports import ObjectContent, ObjectStorage, StoredObject
from messenger.shared.application.errors import Unavailable
class NullLogger:
__slots__ = ()
def info(self, event: str, /, **fields: Any) -> None: ...
def warning(self, event: str, /, **fields: Any) -> None: ...
def error(self, event: str, /, **fields: Any) -> None: ...
class FrozenClock:
__slots__ = ("_current",)
def __init__(self, start: datetime) -> None:
self._current = start
def now(self) -> datetime:
self._current += timedelta(microseconds=1)
return self._current
async def pieces(data: bytes, size: int = 65_536) -> AsyncIterator[bytes]:
for start in range(0, len(data), size):
yield data[start : start + size]
class MemoryObjectStorage:
__slots__ = ("after_page", "after_put", "failing", "modified", "objects")
def __init__(self) -> None:
self.objects: dict[str, tuple[bytes, str]] = {}
self.modified: dict[str, datetime] = {}
self.failing = False
self.after_put: Callable[[str], Awaitable[None]] | None = None
self.after_page: Callable[[], Awaitable[None]] | None = None
async def put(self, key: str, data: bytes, content_type: str) -> None:
if self.failing:
raise Unavailable("S3 недоступно")
self.objects[key] = (bytes(data), content_type)
self.modified[key] = datetime.now(UTC)
if self.after_put is not None:
await self.after_put(key)
async def listing(self, prefix: str, page_size: int) -> AsyncIterator[Sequence[StoredObject]]:
if self.failing:
raise Unavailable("S3 недоступно")
keys = sorted(key for key in self.objects if key.startswith(prefix))
for start in range(0, len(keys), page_size):
yield [self._stored(key) for key in keys[start : start + page_size]]
if self.after_page is not None:
await self.after_page()
async def stat(self, key: str) -> StoredObject | None:
return self._stored(key) if key in self.objects else None
def _stored(self, key: str) -> StoredObject:
return StoredObject(key, len(self.objects[key][0]), self.modified[key])
async def open(self, key: str) -> ObjectContent:
if self.failing:
raise Unavailable("S3 недоступно")
if key not in self.objects:
raise ObjectMissingError(key)
data = self.objects[key][0]
return ObjectContent(size=len(data), chunks=pieces(data))
async def delete(self, key: str) -> None:
self.objects.pop(key, None)
self.modified.pop(key, None)
class MemoryStorageProvider(Provider):
@provide(scope=Scope.APP)
def storage(self) -> ObjectStorage:
return MemoryObjectStorage()