Files
mtusi2026lab4/student-tests/api/test_reconciler.py
T
ArtyomandClaude Opus 5.5 91731fd317 Reconcile attachment metadata with the bucket in one merge pass
StorageReconciler walks the sorted bucket listing and keyset pages of
attachments ordered by object key side by side, so a pass is linear
and keeps one page of each side in memory. An object without a row is
deleted only when it is older than grace plus skew and a lookup by key
right before the delete still finds no row; a ready row whose object
is missing, has another size or, with the deep check, another digest
is put in quarantine after a HEAD recheck and is never deleted; a stale
intent is claimed with SKIP LOCKED and dropped object first; a
quarantined row whose object matches again is released. Deep checks
hash objects in a bounded task group, each action uses its own unit of
work, and failures of single actions are counted instead of aborting.
An advisory lock lets one pass run at a time. Content of a quarantined
attachment answers 404 without touching S3, its metadata is unchanged.

The storage port gains listing pages and HEAD, the repository keyset
pages, lookup by key, claim and quarantine marking; the platform logger
can write to another stream. The one-off entry point
python -m messenger.attachments.entrypoints.reconcile is a dry run
unless --apply is given, assembles the adapters with the API key of S3
and prints a JSON report with counts per class, samples and the cost
of the pass to stdout while the journal goes to stderr.

Tests run the reconciler against its own migrated database with every
class, apply and repeat, the row that appears after the listing, the
busy lock and the release, and check paged byte-order listing and HEAD
against RustFS.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-29 03:12:10 +03:00

274 lines
10 KiB
Python

import asyncio
import hashlib
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
import asyncpg
import pytest
from httpx import AsyncClient
from test_attachments import storage, trio, upload
from conftest import actor, keyed, running_app
from fakes import MemoryObjectStorage, NullLogger
from messenger.attachments.application.commands import ReconcileStorage
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
from messenger.attachments.entrypoints.reconcile import LOCK_KEY
from messenger.attachments.infrastructure.postgres import (
PostgresAttachmentsUnitOfWork,
PostgresSingleRun,
)
from messenger.migrate import migrate
from messenger.shared.infrastructure.clock import MonotonicUtcClock
from messenger.shared.infrastructure.postgres import Database
from pictures import SAMPLE
GRACE = timedelta(seconds=60)
OLD = timedelta(hours=2)
AGED_PENDING = """UPDATE attachments SET status = 'pending', updated_at = created_at,
created_at = created_at - interval '2 hours' WHERE id = $1"""
LATE_ROW = """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, width, height, object_key, status, created_at, updated_at)
SELECT gen_random_uuid(), chat_id, owner_id, 'late.png', content_type, size_bytes, sha256,
width, height, $2, 'pending', now(), now()
FROM attachments WHERE id = $1"""
@pytest.fixture
def dsn(database: str) -> str:
name = f"reconcile_{uuid4().hex[:12]}"
async def create() -> None:
connection = await asyncpg.connect(database)
try:
await connection.execute(f"CREATE DATABASE {name}")
finally:
await connection.close()
await migrate(target)
target = f"{database.rsplit('/', 1)[0]}/{name}"
asyncio.run(create())
return target
@dataclass(slots=True)
class Stand:
http: AsyncClient
store: MemoryObjectStorage
database: Database
users: tuple[dict[str, Any], ...]
async def run(self, *, apply: bool = False, deep: bool = False) -> Reconciliation:
reconciler = StorageReconciler(
lambda: PostgresAttachmentsUnitOfWork(self.database),
self.store,
PostgresSingleRun(self.database, LOCK_KEY),
MonotonicUtcClock(),
NullLogger(),
)
command = ReconcileStorage(
apply=apply, grace=GRACE, skew=timedelta(0), deep=deep, page_size=2
)
return await reconciler(command)
async def sql(self, query: str, *args: Any) -> Any:
async with self.database.connection() as connection:
return await connection.fetch(query, *args)
async def upload(self) -> dict[str, Any]:
alice, _, _, chat = self.users
response = await upload(self.http, alice, chat)
assert response.status_code == 201, response.text
return response.json()
def age(self, key: str) -> None:
self.store.modified[key] = datetime.now(UTC) - OLD
def state(self) -> dict[str, bytes]:
return {key: data for key, (data, _) in self.store.objects.items()}
@asynccontextmanager
async def stand(dsn: str) -> AsyncIterator[Stand]:
async with running_app(database_url=dsn) as http:
database = Database(
dsn, max_size=12, connect_timeout=2, acquire_timeout=2, command_timeout=5
)
await database.open()
try:
yield Stand(http, await storage(http), database, await trio(http))
finally:
await database.close()
def key(attachment: dict[str, Any]) -> str:
return f"attachments/{attachment['id']}"
async def damage(s: Stand) -> dict[str, Any]:
alice, _, _, chat = s.users
healthy, missing, sized, flipped, stale, fresh = [await s.upload() for _ in range(6)]
posted = await s.http.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "", "attachment_ids": [missing["id"]]},
headers=keyed(alice),
)
assert posted.status_code == 201, posted.text
del s.store.objects[key(missing)]
s.store.objects[key(sized)] = (SAMPLE[:100], "image/png")
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
await s.sql(AGED_PENDING, stale["id"])
await s.sql("UPDATE attachments SET status = 'pending' WHERE id = $1", fresh["id"])
orphan, newborn = f"attachments/{uuid4()}", f"attachments/{uuid4()}"
s.store.objects[orphan] = s.store.objects[newborn] = (SAMPLE, "image/png")
s.store.modified[newborn] = datetime.now(UTC)
s.age(orphan)
return {
"healthy": healthy,
"missing": missing,
"sized": sized,
"flipped": flipped,
"stale": stale,
"fresh": fresh,
"orphan": orphan,
"newborn": newborn,
"message": posted.json(),
}
EVERY_CLASS = {
"orphan_stale": 1,
"orphan_fresh": 1,
"missing": 1,
"size_mismatch": 1,
"sha_mismatch": 1,
"pending_stale": 1,
"pending_fresh": 1,
}
async def test_dry_run_reports_every_class_and_changes_nothing(dsn: str) -> None:
async with stand(dsn) as s:
await damage(s)
objects = s.state()
rows = await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id")
shallow = await s.run()
deep = await s.run(deep=True)
assert s.state() == objects
assert (
await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id") == rows
)
assert dict(deep.found) == EVERY_CLASS
assert dict(shallow.found) == {k: v for k, v in EVERY_CLASS.items() if k != "sha_mismatch"}
assert not deep.fixed
assert not shallow.fixed
assert shallow.requests == {"s3_list": 4, "sql": 4}
assert deep.requests["s3_get"] == 2
async def test_apply_fixes_what_it_found_and_repeats_as_noop(dsn: str) -> None:
async with stand(dsn) as s:
damaged = await damage(s)
alice, bob, _, chat = s.users
first = await s.run(apply=True, deep=True)
second = await s.run(apply=True, deep=True)
reasons = {
str(row["id"]): row["quarantine_reason"]
for row in await s.sql("SELECT id, quarantine_reason FROM attachments")
}
history = await s.http.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(bob))
meta = await s.http.get(
f"/api/v1/attachments/{damaged['flipped']['id']}", headers=actor(bob)
)
broken = await s.http.get(
f"/api/v1/attachments/{damaged['flipped']['id']}/content", headers=actor(bob)
)
intact = await s.http.get(
f"/api/v1/attachments/{damaged['healthy']['id']}/content", headers=actor(alice)
)
objects = s.state()
assert {name: first.fixed[name] for name in EVERY_CLASS} == {
**EVERY_CLASS,
"orphan_fresh": 0,
"pending_fresh": 0,
}
assert not second.fixed
assert dict(second.found) == {"orphan_fresh": 1, "pending_fresh": 1, "quarantined": 3}
assert reasons[damaged["missing"]["id"]] == "missing"
assert reasons[damaged["sized"]["id"]] == "size_mismatch"
assert reasons[damaged["flipped"]["id"]] == "sha_mismatch"
assert reasons[damaged["healthy"]["id"]] is None
assert reasons[damaged["fresh"]["id"]] is None
assert damaged["stale"]["id"] not in reasons
assert key(damaged["stale"]) not in objects
assert damaged["orphan"] not in objects
assert damaged["newborn"] in objects
assert key(damaged["flipped"]) in objects
assert history.json()["items"][0]["attachment_ids"] == [damaged["missing"]["id"]]
assert meta.status_code == 200
assert meta.json()["sha256"] == damaged["flipped"]["sha256"]
assert broken.status_code == 404
assert broken.json()["error"]["code"] == "not_found"
assert intact.status_code == 200
assert intact.content == SAMPLE
async def test_restored_object_leaves_quarantine(dsn: str) -> None:
async with stand(dsn) as s:
_, bob, _, _ = s.users
lost, flipped = await s.upload(), await s.upload()
del s.store.objects[key(lost)]
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
isolated = await s.run(apply=True, deep=True)
s.store.objects[key(lost)] = s.store.objects[key(flipped)] = (SAMPLE, "image/png")
s.store.modified[key(lost)] = s.store.modified[key(flipped)]
shallow = await s.run(apply=True)
deep = await s.run(apply=True, deep=True)
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
assert isolated.fixed["missing"] == isolated.fixed["sha_mismatch"] == 1
assert shallow.fixed["healed"] == 1
assert shallow.found["quarantined"] == 1
assert deep.fixed["healed"] == 1
assert hashlib.sha256(content.content).hexdigest() == lost["sha256"]
async def test_row_written_after_listing_keeps_the_object(dsn: str) -> None:
async with stand(dsn) as s:
first, last = await s.upload(), await s.upload()
fresh, late = f"{key(first)}-a", f"{key(first)}-b"
for name in (fresh, late):
s.store.objects[name] = (SAMPLE, "image/png")
s.store.modified[name] = datetime.now(UTC)
s.age(late)
async def row_appears() -> None:
s.store.after_page = None
await s.sql(LATE_ROW, first["id"], late)
s.store.after_page = row_appears
report = await s.run(apply=True)
objects = s.state()
assert key(first) < fresh < late < key(last)
assert report.found["orphan_stale"] == 1
assert report.rechecked["orphan_stale"] == 1
assert not report.fixed
assert late in objects
async def test_second_pass_is_skipped_while_one_runs(dsn: str) -> None:
async with stand(dsn) as s:
orphan = f"attachments/{uuid4()}"
s.store.objects[orphan] = (SAMPLE, "image/png")
s.age(orphan)
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
busy = await s.run(apply=True)
free = await s.run(apply=True)
assert held
assert busy.skipped
assert not busy.found
assert not free.skipped
assert free.fixed["orphan_stale"] == 1