Carry sourcef591fa64e5forward from72a1e57. Preserve lab4 attachment transactions, reconciliation locks, quarantine, and existing schema data through verified legacy adoption. Keep report and evidence work in progress untouched.
391 lines
15 KiB
Python
391 lines
15 KiB
Python
import asyncio
|
|
import hashlib
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from sqlalchemy import make_url
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from sqlalchemy.pool import NullPool
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import actor, keyed, running_app, upgrade
|
|
from fakes import MemoryObjectStorage, NullLogger, RecordingPublisher
|
|
from messenger.attachments.application.commands import ReconcileStorage
|
|
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
|
|
from messenger.attachments.domain.events import AttachmentDiscarded
|
|
from messenger.attachments.entrypoints import reconcile as entrypoint
|
|
from messenger.attachments.entrypoints.reconcile import LOCK_KEY
|
|
from messenger.attachments.infrastructure.postgres import (
|
|
PostgresAttachmentsUnitOfWork,
|
|
PostgresSingleRun,
|
|
)
|
|
from messenger.shared.application.errors import Unavailable
|
|
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
|
from messenger.shared.infrastructure.postgres import Database
|
|
from pictures import SAMPLE
|
|
|
|
GRACE = timedelta(seconds=60)
|
|
OLD = timedelta(hours=2)
|
|
AGED_PENDING = """UPDATE attachments SET status = 'pending', updated_at = created_at,
|
|
created_at = created_at - interval '2 hours' WHERE id = $1"""
|
|
LATE_ROW = """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes,
|
|
sha256, width, height, object_key, status, created_at, updated_at)
|
|
SELECT gen_random_uuid(), chat_id, owner_id, 'late.png', content_type, size_bytes, sha256,
|
|
width, height, $2, 'pending', now(), now()
|
|
FROM attachments WHERE id = $1"""
|
|
|
|
|
|
@pytest.fixture
|
|
def dsn(database: str) -> str:
|
|
name = f"reconcile_{uuid4().hex[:12]}"
|
|
|
|
async def create() -> None:
|
|
engine = create_async_engine(
|
|
make_url(database).set(drivername="postgresql+asyncpg"),
|
|
isolation_level="AUTOCOMMIT",
|
|
poolclass=NullPool,
|
|
)
|
|
try:
|
|
async with engine.connect() as connection:
|
|
await connection.exec_driver_sql(f"CREATE DATABASE {name}")
|
|
finally:
|
|
await engine.dispose()
|
|
await upgrade(target)
|
|
|
|
target = f"{database.rsplit('/', 1)[0]}/{name}"
|
|
asyncio.run(create())
|
|
return target
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Stand:
|
|
http: AsyncClient
|
|
store: MemoryObjectStorage
|
|
database: Database
|
|
users: tuple[dict[str, Any], ...]
|
|
published: RecordingPublisher
|
|
|
|
async def run(self, *, apply: bool = False, deep: bool = False) -> Reconciliation:
|
|
reconciler = StorageReconciler(
|
|
lambda: PostgresAttachmentsUnitOfWork(self.database),
|
|
self.store,
|
|
PostgresSingleRun(self.database, LOCK_KEY),
|
|
MonotonicUtcClock(),
|
|
logger=NullLogger(),
|
|
publisher=self.published,
|
|
)
|
|
command = ReconcileStorage(
|
|
apply=apply, grace=GRACE, skew=timedelta(0), deep=deep, page_size=2
|
|
)
|
|
return await reconciler(command)
|
|
|
|
async def sql(self, query: str, *args: Any) -> Any:
|
|
async with self.database.session() as session:
|
|
connection = await session.connection()
|
|
result = await connection.exec_driver_sql(query, args)
|
|
rows = result.mappings().all() if result.returns_rows else []
|
|
await session.commit()
|
|
return rows
|
|
|
|
async def upload(self) -> dict[str, Any]:
|
|
alice, _, _, chat = self.users
|
|
response = await upload(self.http, alice, chat)
|
|
assert response.status_code == 201, response.text
|
|
return response.json()
|
|
|
|
def age(self, key: str) -> None:
|
|
self.store.modified[key] = datetime.now(UTC) - OLD
|
|
|
|
def state(self) -> dict[str, bytes]:
|
|
return {key: data for key, (data, _) in self.store.objects.items()}
|
|
|
|
|
|
@asynccontextmanager
|
|
async def stand(dsn: str) -> AsyncIterator[Stand]:
|
|
async with running_app(database_url=dsn) as http:
|
|
database = Database(
|
|
dsn, max_size=12, connect_timeout=2, acquire_timeout=2, command_timeout=5
|
|
)
|
|
await database.open()
|
|
try:
|
|
yield Stand(http, await storage(http), database, await trio(http), RecordingPublisher())
|
|
finally:
|
|
await database.close()
|
|
|
|
|
|
def key(attachment: dict[str, Any]) -> str:
|
|
return f"attachments/{attachment['id']}"
|
|
|
|
|
|
async def damage(s: Stand) -> dict[str, Any]:
|
|
alice, _, _, chat = s.users
|
|
healthy, missing, sized, flipped, stale, fresh = [await s.upload() for _ in range(6)]
|
|
posted = await s.http.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [missing["id"]]},
|
|
headers=keyed(alice),
|
|
)
|
|
assert posted.status_code == 201, posted.text
|
|
del s.store.objects[key(missing)]
|
|
s.store.objects[key(sized)] = (SAMPLE[:100], "image/png")
|
|
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
|
|
await s.sql(AGED_PENDING, stale["id"])
|
|
await s.sql("UPDATE attachments SET status = 'pending' WHERE id = $1", fresh["id"])
|
|
orphan, newborn = f"attachments/{uuid4()}", f"attachments/{uuid4()}"
|
|
s.store.objects[orphan] = s.store.objects[newborn] = (SAMPLE, "image/png")
|
|
s.store.modified[newborn] = datetime.now(UTC)
|
|
s.age(orphan)
|
|
return {
|
|
"healthy": healthy,
|
|
"missing": missing,
|
|
"sized": sized,
|
|
"flipped": flipped,
|
|
"stale": stale,
|
|
"fresh": fresh,
|
|
"orphan": orphan,
|
|
"newborn": newborn,
|
|
"message": posted.json(),
|
|
}
|
|
|
|
|
|
EVERY_CLASS = {
|
|
"orphan_stale": 1,
|
|
"orphan_fresh": 1,
|
|
"missing": 1,
|
|
"size_mismatch": 1,
|
|
"sha_mismatch": 1,
|
|
"pending_stale": 1,
|
|
"pending_fresh": 1,
|
|
}
|
|
|
|
|
|
async def test_dry_run_reports_every_class_and_changes_nothing(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
await damage(s)
|
|
objects = s.state()
|
|
rows = await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id")
|
|
shallow = await s.run()
|
|
deep = await s.run(deep=True)
|
|
assert s.state() == objects
|
|
assert (
|
|
await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id") == rows
|
|
)
|
|
assert dict(deep.found) == EVERY_CLASS
|
|
assert dict(shallow.found) == {k: v for k, v in EVERY_CLASS.items() if k != "sha_mismatch"}
|
|
assert not deep.fixed
|
|
assert not shallow.fixed
|
|
assert shallow.requests == {"s3_list": 4, "sql": 3}
|
|
assert deep.requests["s3_get"] == 2
|
|
|
|
|
|
async def test_apply_fixes_what_it_found_and_repeats_as_noop(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
damaged = await damage(s)
|
|
alice, bob, _, chat = s.users
|
|
first = await s.run(apply=True, deep=True)
|
|
second = await s.run(apply=True, deep=True)
|
|
reasons = {
|
|
str(row["id"]): row["quarantine_reason"]
|
|
for row in await s.sql("SELECT id, quarantine_reason FROM attachments")
|
|
}
|
|
history = await s.http.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(bob))
|
|
meta = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['flipped']['id']}", headers=actor(bob)
|
|
)
|
|
broken = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['flipped']['id']}/content", headers=actor(bob)
|
|
)
|
|
intact = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['healthy']['id']}/content", headers=actor(alice)
|
|
)
|
|
objects = s.state()
|
|
assert {name: first.fixed[name] for name in EVERY_CLASS} == {
|
|
**EVERY_CLASS,
|
|
"orphan_fresh": 0,
|
|
"pending_fresh": 0,
|
|
}
|
|
assert not second.fixed
|
|
assert dict(second.found) == {"orphan_fresh": 1, "pending_fresh": 1, "quarantined": 3}
|
|
assert reasons[damaged["missing"]["id"]] == "missing"
|
|
assert reasons[damaged["sized"]["id"]] == "size_mismatch"
|
|
assert reasons[damaged["flipped"]["id"]] == "sha_mismatch"
|
|
assert reasons[damaged["healthy"]["id"]] is None
|
|
assert [
|
|
(event.attachment_id, event.reason)
|
|
for event in s.published.events
|
|
if isinstance(event, AttachmentDiscarded)
|
|
] == [(UUID(damaged["stale"]["id"]), "reconcile")]
|
|
assert reasons[damaged["fresh"]["id"]] is None
|
|
assert damaged["stale"]["id"] not in reasons
|
|
assert key(damaged["stale"]) not in objects
|
|
assert damaged["orphan"] not in objects
|
|
assert damaged["newborn"] in objects
|
|
assert key(damaged["flipped"]) in objects
|
|
assert history.json()["items"][0]["attachment_ids"] == [damaged["missing"]["id"]]
|
|
assert meta.status_code == 200
|
|
assert meta.json()["sha256"] == damaged["flipped"]["sha256"]
|
|
assert broken.status_code == 404
|
|
assert broken.json()["error"]["code"] == "not_found"
|
|
assert intact.status_code == 200
|
|
assert intact.content == SAMPLE
|
|
|
|
|
|
async def test_restored_object_leaves_quarantine(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
_, bob, _, _ = s.users
|
|
lost, flipped = await s.upload(), await s.upload()
|
|
del s.store.objects[key(lost)]
|
|
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
|
|
isolated = await s.run(apply=True, deep=True)
|
|
s.store.objects[key(lost)] = s.store.objects[key(flipped)] = (SAMPLE, "image/png")
|
|
s.store.modified[key(lost)] = s.store.modified[key(flipped)]
|
|
shallow = await s.run(apply=True)
|
|
again = await s.run(apply=True)
|
|
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
|
|
assert isolated.fixed["missing"] == isolated.fixed["sha_mismatch"] == 1
|
|
assert shallow.fixed["healed"] == 2
|
|
assert shallow.requests["s3_get"] == 2
|
|
assert not again.fixed
|
|
assert again.requests["s3_get"] == 0
|
|
assert hashlib.sha256(content.content).hexdigest() == lost["sha256"]
|
|
|
|
|
|
async def test_other_bytes_of_the_same_length_keep_the_quarantine(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
_, bob, _, _ = s.users
|
|
lost = await s.upload()
|
|
del s.store.objects[key(lost)]
|
|
isolated = await s.run(apply=True)
|
|
s.store.objects[key(lost)] = (bytes(len(SAMPLE)), "image/png")
|
|
s.age(key(lost))
|
|
shallow = await s.run(apply=True)
|
|
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
|
|
reason = await s.sql("SELECT quarantine_reason FROM attachments WHERE id = $1", lost["id"])
|
|
assert isolated.fixed["missing"] == 1
|
|
assert not shallow.fixed
|
|
assert shallow.found["quarantined"] == 1
|
|
assert shallow.requests["s3_get"] == 1
|
|
assert content.status_code == 404
|
|
assert reason[0]["quarantine_reason"] == "missing"
|
|
|
|
|
|
async def test_pages_of_rows_are_not_followed_by_an_empty_one(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
empty = await s.run()
|
|
for _ in range(4):
|
|
await s.upload()
|
|
full = await s.run()
|
|
assert (empty.rows, empty.requests["sql"]) == (0, 1)
|
|
assert (full.rows, full.requests["sql"]) == (4, 2)
|
|
|
|
|
|
def test_any_pass_failure_ends_as_reconcile_failed(
|
|
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
async def broken(*_args: Any) -> Reconciliation:
|
|
cancelled = TimeoutError("statement timeout")
|
|
raise ExceptionGroup("pass", [cancelled])
|
|
|
|
monkeypatch.setattr(entrypoint, "reconcile", broken)
|
|
for name, value in {
|
|
"DATABASE_URL": "postgresql://unused@127.0.0.1:1/unused",
|
|
"S3_ACCESS_KEY_ID": "reconciler",
|
|
"S3_SECRET_ACCESS_KEY": "reconciler-secret",
|
|
}.items():
|
|
monkeypatch.setenv(name, value)
|
|
with pytest.raises(SystemExit) as stopped:
|
|
entrypoint.main(["--apply"])
|
|
captured = capsys.readouterr()
|
|
assert stopped.value.code == 1
|
|
assert captured.out == ""
|
|
assert "reconcile.failed" in captured.err
|
|
assert "TimeoutError" in captured.err
|
|
assert "Traceback" not in captured.err
|
|
|
|
|
|
async def test_row_written_after_listing_keeps_the_object(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
first, last = await s.upload(), await s.upload()
|
|
fresh, late = f"{key(first)}-a", f"{key(first)}-b"
|
|
for name in (fresh, late):
|
|
s.store.objects[name] = (SAMPLE, "image/png")
|
|
s.store.modified[name] = datetime.now(UTC)
|
|
s.age(late)
|
|
|
|
async def row_appears() -> None:
|
|
s.store.after_page = None
|
|
await s.sql(LATE_ROW, first["id"], late)
|
|
|
|
s.store.after_page = row_appears
|
|
report = await s.run(apply=True)
|
|
objects = s.state()
|
|
assert key(first) < fresh < late < key(last)
|
|
assert report.found["orphan_stale"] == 1
|
|
assert report.rechecked["orphan_stale"] == 1
|
|
assert not report.fixed
|
|
assert late in objects
|
|
|
|
|
|
async def test_second_pass_is_skipped_while_one_runs(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
orphan = f"attachments/{uuid4()}"
|
|
s.store.objects[orphan] = (SAMPLE, "image/png")
|
|
s.age(orphan)
|
|
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
|
|
busy = await s.run(apply=True)
|
|
free = await s.run(apply=True)
|
|
assert held
|
|
assert busy.skipped
|
|
assert not busy.found
|
|
assert not free.skipped
|
|
assert free.fixed["orphan_stale"] == 1
|
|
|
|
|
|
async def test_storage_failure_mid_pass_ends_it_as_unavailable(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
await damage(s)
|
|
|
|
async def broken() -> None:
|
|
raise Unavailable("S3 недоступно") from TimeoutError()
|
|
|
|
s.store.after_page = broken
|
|
with pytest.raises(Unavailable) as failed:
|
|
await s.run(apply=True, deep=True)
|
|
assert isinstance(failed.value.__cause__, TimeoutError)
|
|
|
|
|
|
async def test_cancelled_pass_releases_its_autocommit_lock(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
entered = asyncio.Event()
|
|
|
|
async def hold() -> None:
|
|
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
|
|
assert held
|
|
entered.set()
|
|
await asyncio.Event().wait()
|
|
|
|
task = asyncio.create_task(hold())
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 2)
|
|
states = await s.sql(
|
|
"SELECT a.state FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid "
|
|
"WHERE l.locktype = 'advisory' AND l.objid = $1",
|
|
LOCK_KEY,
|
|
)
|
|
assert [row["state"] for row in states] == ["idle"]
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
locks = await s.sql(
|
|
"SELECT count(*) AS total FROM pg_locks WHERE locktype = 'advisory' AND objid = $1",
|
|
LOCK_KEY,
|
|
)
|
|
assert locks[0]["total"] == 0
|