forked from mtusi2026/mtusi2026lab2
StorageReconciler walks the sorted bucket listing and keyset pages of attachments ordered by object key side by side, so a pass is linear and keeps one page of each side in memory. An object without a row is deleted only when it is older than grace plus skew and a lookup by key right before the delete still finds no row; a ready row whose object is missing, has another size or, with the deep check, another digest is put in quarantine after a HEAD recheck and is never deleted; a stale intent is claimed with SKIP LOCKED and dropped object first; a quarantined row whose object matches again is released. Deep checks hash objects in a bounded task group, each action uses its own unit of work, and failures of single actions are counted instead of aborting. An advisory lock lets one pass run at a time. Content of a quarantined attachment answers 404 without touching S3, its metadata is unchanged. The storage port gains listing pages and HEAD, the repository keyset pages, lookup by key, claim and quarantine marking; the platform logger can write to another stream. The one-off entry point python -m messenger.attachments.entrypoints.reconcile is a dry run unless --apply is given, assembles the adapters with the API key of S3 and prints a JSON report with counts per class, samples and the cost of the pass to stdout while the journal goes to stderr. Tests run the reconciler against its own migrated database with every class, apply and repeat, the row that appears after the listing, the busy lock and the release, and check paged byte-order listing and HEAD against RustFS. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
274 lines
10 KiB
Python
274 lines
10 KiB
Python
import asyncio
|
|
import hashlib
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import actor, keyed, running_app
|
|
from fakes import MemoryObjectStorage, NullLogger
|
|
from messenger.attachments.application.commands import ReconcileStorage
|
|
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
|
|
from messenger.attachments.entrypoints.reconcile import LOCK_KEY
|
|
from messenger.attachments.infrastructure.postgres import (
|
|
PostgresAttachmentsUnitOfWork,
|
|
PostgresSingleRun,
|
|
)
|
|
from messenger.migrate import migrate
|
|
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
|
from messenger.shared.infrastructure.postgres import Database
|
|
from pictures import SAMPLE
|
|
|
|
GRACE = timedelta(seconds=60)
|
|
OLD = timedelta(hours=2)
|
|
AGED_PENDING = """UPDATE attachments SET status = 'pending', updated_at = created_at,
|
|
created_at = created_at - interval '2 hours' WHERE id = $1"""
|
|
LATE_ROW = """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes,
|
|
sha256, width, height, object_key, status, created_at, updated_at)
|
|
SELECT gen_random_uuid(), chat_id, owner_id, 'late.png', content_type, size_bytes, sha256,
|
|
width, height, $2, 'pending', now(), now()
|
|
FROM attachments WHERE id = $1"""
|
|
|
|
|
|
@pytest.fixture
|
|
def dsn(database: str) -> str:
|
|
name = f"reconcile_{uuid4().hex[:12]}"
|
|
|
|
async def create() -> None:
|
|
connection = await asyncpg.connect(database)
|
|
try:
|
|
await connection.execute(f"CREATE DATABASE {name}")
|
|
finally:
|
|
await connection.close()
|
|
await migrate(target)
|
|
|
|
target = f"{database.rsplit('/', 1)[0]}/{name}"
|
|
asyncio.run(create())
|
|
return target
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Stand:
|
|
http: AsyncClient
|
|
store: MemoryObjectStorage
|
|
database: Database
|
|
users: tuple[dict[str, Any], ...]
|
|
|
|
async def run(self, *, apply: bool = False, deep: bool = False) -> Reconciliation:
|
|
reconciler = StorageReconciler(
|
|
lambda: PostgresAttachmentsUnitOfWork(self.database),
|
|
self.store,
|
|
PostgresSingleRun(self.database, LOCK_KEY),
|
|
MonotonicUtcClock(),
|
|
NullLogger(),
|
|
)
|
|
command = ReconcileStorage(
|
|
apply=apply, grace=GRACE, skew=timedelta(0), deep=deep, page_size=2
|
|
)
|
|
return await reconciler(command)
|
|
|
|
async def sql(self, query: str, *args: Any) -> Any:
|
|
async with self.database.connection() as connection:
|
|
return await connection.fetch(query, *args)
|
|
|
|
async def upload(self) -> dict[str, Any]:
|
|
alice, _, _, chat = self.users
|
|
response = await upload(self.http, alice, chat)
|
|
assert response.status_code == 201, response.text
|
|
return response.json()
|
|
|
|
def age(self, key: str) -> None:
|
|
self.store.modified[key] = datetime.now(UTC) - OLD
|
|
|
|
def state(self) -> dict[str, bytes]:
|
|
return {key: data for key, (data, _) in self.store.objects.items()}
|
|
|
|
|
|
@asynccontextmanager
|
|
async def stand(dsn: str) -> AsyncIterator[Stand]:
|
|
async with running_app(database_url=dsn) as http:
|
|
database = Database(
|
|
dsn, max_size=12, connect_timeout=2, acquire_timeout=2, command_timeout=5
|
|
)
|
|
await database.open()
|
|
try:
|
|
yield Stand(http, await storage(http), database, await trio(http))
|
|
finally:
|
|
await database.close()
|
|
|
|
|
|
def key(attachment: dict[str, Any]) -> str:
|
|
return f"attachments/{attachment['id']}"
|
|
|
|
|
|
async def damage(s: Stand) -> dict[str, Any]:
|
|
alice, _, _, chat = s.users
|
|
healthy, missing, sized, flipped, stale, fresh = [await s.upload() for _ in range(6)]
|
|
posted = await s.http.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [missing["id"]]},
|
|
headers=keyed(alice),
|
|
)
|
|
assert posted.status_code == 201, posted.text
|
|
del s.store.objects[key(missing)]
|
|
s.store.objects[key(sized)] = (SAMPLE[:100], "image/png")
|
|
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
|
|
await s.sql(AGED_PENDING, stale["id"])
|
|
await s.sql("UPDATE attachments SET status = 'pending' WHERE id = $1", fresh["id"])
|
|
orphan, newborn = f"attachments/{uuid4()}", f"attachments/{uuid4()}"
|
|
s.store.objects[orphan] = s.store.objects[newborn] = (SAMPLE, "image/png")
|
|
s.store.modified[newborn] = datetime.now(UTC)
|
|
s.age(orphan)
|
|
return {
|
|
"healthy": healthy,
|
|
"missing": missing,
|
|
"sized": sized,
|
|
"flipped": flipped,
|
|
"stale": stale,
|
|
"fresh": fresh,
|
|
"orphan": orphan,
|
|
"newborn": newborn,
|
|
"message": posted.json(),
|
|
}
|
|
|
|
|
|
EVERY_CLASS = {
|
|
"orphan_stale": 1,
|
|
"orphan_fresh": 1,
|
|
"missing": 1,
|
|
"size_mismatch": 1,
|
|
"sha_mismatch": 1,
|
|
"pending_stale": 1,
|
|
"pending_fresh": 1,
|
|
}
|
|
|
|
|
|
async def test_dry_run_reports_every_class_and_changes_nothing(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
await damage(s)
|
|
objects = s.state()
|
|
rows = await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id")
|
|
shallow = await s.run()
|
|
deep = await s.run(deep=True)
|
|
assert s.state() == objects
|
|
assert (
|
|
await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id") == rows
|
|
)
|
|
assert dict(deep.found) == EVERY_CLASS
|
|
assert dict(shallow.found) == {k: v for k, v in EVERY_CLASS.items() if k != "sha_mismatch"}
|
|
assert not deep.fixed
|
|
assert not shallow.fixed
|
|
assert shallow.requests == {"s3_list": 4, "sql": 4}
|
|
assert deep.requests["s3_get"] == 2
|
|
|
|
|
|
async def test_apply_fixes_what_it_found_and_repeats_as_noop(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
damaged = await damage(s)
|
|
alice, bob, _, chat = s.users
|
|
first = await s.run(apply=True, deep=True)
|
|
second = await s.run(apply=True, deep=True)
|
|
reasons = {
|
|
str(row["id"]): row["quarantine_reason"]
|
|
for row in await s.sql("SELECT id, quarantine_reason FROM attachments")
|
|
}
|
|
history = await s.http.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(bob))
|
|
meta = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['flipped']['id']}", headers=actor(bob)
|
|
)
|
|
broken = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['flipped']['id']}/content", headers=actor(bob)
|
|
)
|
|
intact = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['healthy']['id']}/content", headers=actor(alice)
|
|
)
|
|
objects = s.state()
|
|
assert {name: first.fixed[name] for name in EVERY_CLASS} == {
|
|
**EVERY_CLASS,
|
|
"orphan_fresh": 0,
|
|
"pending_fresh": 0,
|
|
}
|
|
assert not second.fixed
|
|
assert dict(second.found) == {"orphan_fresh": 1, "pending_fresh": 1, "quarantined": 3}
|
|
assert reasons[damaged["missing"]["id"]] == "missing"
|
|
assert reasons[damaged["sized"]["id"]] == "size_mismatch"
|
|
assert reasons[damaged["flipped"]["id"]] == "sha_mismatch"
|
|
assert reasons[damaged["healthy"]["id"]] is None
|
|
assert reasons[damaged["fresh"]["id"]] is None
|
|
assert damaged["stale"]["id"] not in reasons
|
|
assert key(damaged["stale"]) not in objects
|
|
assert damaged["orphan"] not in objects
|
|
assert damaged["newborn"] in objects
|
|
assert key(damaged["flipped"]) in objects
|
|
assert history.json()["items"][0]["attachment_ids"] == [damaged["missing"]["id"]]
|
|
assert meta.status_code == 200
|
|
assert meta.json()["sha256"] == damaged["flipped"]["sha256"]
|
|
assert broken.status_code == 404
|
|
assert broken.json()["error"]["code"] == "not_found"
|
|
assert intact.status_code == 200
|
|
assert intact.content == SAMPLE
|
|
|
|
|
|
async def test_restored_object_leaves_quarantine(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
_, bob, _, _ = s.users
|
|
lost, flipped = await s.upload(), await s.upload()
|
|
del s.store.objects[key(lost)]
|
|
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
|
|
isolated = await s.run(apply=True, deep=True)
|
|
s.store.objects[key(lost)] = s.store.objects[key(flipped)] = (SAMPLE, "image/png")
|
|
s.store.modified[key(lost)] = s.store.modified[key(flipped)]
|
|
shallow = await s.run(apply=True)
|
|
deep = await s.run(apply=True, deep=True)
|
|
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
|
|
assert isolated.fixed["missing"] == isolated.fixed["sha_mismatch"] == 1
|
|
assert shallow.fixed["healed"] == 1
|
|
assert shallow.found["quarantined"] == 1
|
|
assert deep.fixed["healed"] == 1
|
|
assert hashlib.sha256(content.content).hexdigest() == lost["sha256"]
|
|
|
|
|
|
async def test_row_written_after_listing_keeps_the_object(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
first, last = await s.upload(), await s.upload()
|
|
fresh, late = f"{key(first)}-a", f"{key(first)}-b"
|
|
for name in (fresh, late):
|
|
s.store.objects[name] = (SAMPLE, "image/png")
|
|
s.store.modified[name] = datetime.now(UTC)
|
|
s.age(late)
|
|
|
|
async def row_appears() -> None:
|
|
s.store.after_page = None
|
|
await s.sql(LATE_ROW, first["id"], late)
|
|
|
|
s.store.after_page = row_appears
|
|
report = await s.run(apply=True)
|
|
objects = s.state()
|
|
assert key(first) < fresh < late < key(last)
|
|
assert report.found["orphan_stale"] == 1
|
|
assert report.rechecked["orphan_stale"] == 1
|
|
assert not report.fixed
|
|
assert late in objects
|
|
|
|
|
|
async def test_second_pass_is_skipped_while_one_runs(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
orphan = f"attachments/{uuid4()}"
|
|
s.store.objects[orphan] = (SAMPLE, "image/png")
|
|
s.age(orphan)
|
|
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
|
|
busy = await s.run(apply=True)
|
|
free = await s.run(apply=True)
|
|
assert held
|
|
assert busy.skipped
|
|
assert not busy.found
|
|
assert not free.skipped
|
|
assert free.fixed["orphan_stale"] == 1
|