Files
mtusi2026lab6/student-tests/api/test_reconciler.py
T
Artyom 12db77133f Forward the lab 5 foundation while preserving lab 6 telemetry
Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files.

Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
2026-11-06 12:29:35 +03:00

433 lines
16 KiB
Python

import asyncio
import hashlib
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import UUID, uuid4
import pytest
from httpx import AsyncClient
from sqlalchemy import make_url
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import NullPool
from test_attachments import storage, trio, upload
from conftest import actor, keyed, running_app, upgrade
from fakes import MemoryObjectStorage, NullLogger, RecordingPublisher
from messenger.attachments.application.commands import ReconcileStorage
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
from messenger.attachments.domain.events import AttachmentDiscarded
from messenger.attachments.entrypoints import reconcile as entrypoint
from messenger.attachments.entrypoints.reconcile import LOCK_KEY
from messenger.attachments.infrastructure.postgres import (
PostgresAttachmentsUnitOfWork,
PostgresSingleRun,
)
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.clock import MonotonicUtcClock
from messenger.shared.infrastructure.postgres import Database
from pictures import SAMPLE
from pipeline import settle
GRACE = timedelta(seconds=60)
OLD = timedelta(hours=2)
AGED_PENDING = """UPDATE attachments SET status = 'pending', staging_key = object_key,
original_key = NULL, updated_at = created_at,
created_at = created_at - interval '2 hours' WHERE id = $1"""
LATE_ROW = """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, width, height, object_key, staging_key, status, created_at, updated_at)
SELECT gen_random_uuid(), chat_id, owner_id, 'late.png', content_type, size_bytes, sha256,
width, height, $2, $2, 'pending', now(), now()
FROM attachments WHERE id = $1"""
@pytest.fixture
def dsn(database: str) -> str:
name = f"reconcile_{uuid4().hex[:12]}"
async def create() -> None:
engine = create_async_engine(
make_url(database).set(drivername="postgresql+asyncpg"),
isolation_level="AUTOCOMMIT",
poolclass=NullPool,
)
try:
async with engine.connect() as connection:
await connection.exec_driver_sql(f"CREATE DATABASE {name}")
finally:
await engine.dispose()
await upgrade(target)
target = f"{database.rsplit('/', 1)[0]}/{name}"
asyncio.run(create())
return target
@dataclass(slots=True)
class Stand:
http: AsyncClient
store: MemoryObjectStorage
database: Database
users: tuple[dict[str, Any], ...]
published: RecordingPublisher
async def run(self, *, apply: bool = False, deep: bool = False) -> Reconciliation:
reconciler = StorageReconciler(
lambda: PostgresAttachmentsUnitOfWork(self.database),
self.store,
PostgresSingleRun(self.database, LOCK_KEY),
MonotonicUtcClock(),
logger=NullLogger(),
publisher=self.published,
)
command = ReconcileStorage(
apply=apply, grace=GRACE, skew=timedelta(0), deep=deep, page_size=2
)
return await reconciler(command)
async def sql(self, query: str, *args: Any) -> Any:
async with self.database.session() as session:
connection = await session.connection()
result = await connection.exec_driver_sql(query, args)
rows = result.mappings().all() if result.returns_rows else []
await session.commit()
return rows
async def upload(self) -> dict[str, Any]:
alice, _, _, chat = self.users
response = await upload(self.http, alice, chat)
assert response.status_code == 202, response.text
attachment = response.json()
legacy_key = key(attachment)
staging_key = f"staging/{attachment['id']}"
self.store.objects[legacy_key] = self.store.objects.pop(staging_key)
self.store.modified[legacy_key] = self.store.modified.pop(staging_key)
await self.sql(
"DELETE FROM outbox WHERE job_id IN (SELECT job_id FROM jobs WHERE attachment_id = $1)",
attachment["id"],
)
await self.sql("DELETE FROM jobs WHERE attachment_id = $1", attachment["id"])
await self.sql(
"UPDATE attachments SET status = 'ready', object_key = $2, original_key = $2, "
"staging_key = NULL, published_version = 1 WHERE id = $1",
attachment["id"],
legacy_key,
)
return attachment
def age(self, key: str) -> None:
self.store.modified[key] = datetime.now(UTC) - OLD
def state(self) -> dict[str, bytes]:
return {key: data for key, (data, _) in self.store.objects.items()}
@asynccontextmanager
async def stand(dsn: str) -> AsyncIterator[Stand]:
async with running_app(database_url=dsn) as http:
database = Database(
dsn, max_size=12, connect_timeout=2, acquire_timeout=2, command_timeout=5
)
await database.open()
try:
yield Stand(http, await storage(http), database, await trio(http), RecordingPublisher())
finally:
await database.close()
def key(attachment: dict[str, Any]) -> str:
return f"attachments/{attachment['id']}"
async def damage(s: Stand) -> dict[str, Any]:
alice, _, _, chat = s.users
healthy, missing, sized, flipped, stale, fresh = [await s.upload() for _ in range(6)]
posted = await s.http.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "", "attachment_ids": [missing["id"]]},
headers=keyed(alice),
)
assert posted.status_code == 201, posted.text
del s.store.objects[key(missing)]
s.store.objects[key(sized)] = (SAMPLE[:100], "image/png")
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
await s.sql(AGED_PENDING, stale["id"])
await s.sql(
"UPDATE attachments SET status = 'pending', staging_key = object_key, "
"original_key = NULL WHERE id = $1",
fresh["id"],
)
orphan, newborn = f"attachments/{uuid4()}", f"attachments/{uuid4()}"
s.store.objects[orphan] = s.store.objects[newborn] = (SAMPLE, "image/png")
s.store.modified[newborn] = datetime.now(UTC)
s.age(orphan)
return {
"healthy": healthy,
"missing": missing,
"sized": sized,
"flipped": flipped,
"stale": stale,
"fresh": fresh,
"orphan": orphan,
"newborn": newborn,
"message": posted.json(),
}
EVERY_CLASS = {
"orphan_stale": 1,
"orphan_fresh": 1,
"missing": 1,
"size_mismatch": 1,
"sha_mismatch": 1,
"pending_stale": 1,
"pending_fresh": 1,
}
async def test_dry_run_reports_every_class_and_changes_nothing(dsn: str) -> None:
async with stand(dsn) as s:
await damage(s)
objects = s.state()
rows = await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id")
shallow = await s.run()
deep = await s.run(deep=True)
assert s.state() == objects
assert (
await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id") == rows
)
assert dict(deep.found) == EVERY_CLASS
assert dict(shallow.found) == {k: v for k, v in EVERY_CLASS.items() if k != "sha_mismatch"}
assert not deep.fixed
assert not shallow.fixed
assert shallow.requests == {"s3_list": 4, "sql": 3}
assert deep.requests["s3_get"] == 2
async def test_apply_fixes_what_it_found_and_repeats_as_noop(dsn: str) -> None:
async with stand(dsn) as s:
damaged = await damage(s)
alice, bob, _, chat = s.users
first = await s.run(apply=True, deep=True)
second = await s.run(apply=True, deep=True)
reasons = {
str(row["id"]): row["quarantine_reason"]
for row in await s.sql("SELECT id, quarantine_reason FROM attachments")
}
history = await s.http.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(bob))
meta = await s.http.get(
f"/api/v1/attachments/{damaged['flipped']['id']}", headers=actor(bob)
)
broken = await s.http.get(
f"/api/v1/attachments/{damaged['flipped']['id']}/content", headers=actor(bob)
)
intact = await s.http.get(
f"/api/v1/attachments/{damaged['healthy']['id']}/content", headers=actor(alice)
)
objects = s.state()
assert {name: first.fixed[name] for name in EVERY_CLASS} == {
**EVERY_CLASS,
"orphan_fresh": 0,
"pending_fresh": 0,
}
assert not second.fixed
assert dict(second.found) == {"orphan_fresh": 1, "pending_fresh": 1, "quarantined": 3}
assert reasons[damaged["missing"]["id"]] == "missing"
assert reasons[damaged["sized"]["id"]] == "size_mismatch"
assert reasons[damaged["flipped"]["id"]] == "sha_mismatch"
assert reasons[damaged["healthy"]["id"]] is None
assert [
(event.attachment_id, event.reason)
for event in s.published.events
if isinstance(event, AttachmentDiscarded)
] == [(UUID(damaged["stale"]["id"]), "reconcile")]
assert reasons[damaged["fresh"]["id"]] is None
assert damaged["stale"]["id"] not in reasons
assert key(damaged["stale"]) not in objects
assert damaged["orphan"] not in objects
assert damaged["newborn"] in objects
assert key(damaged["flipped"]) in objects
assert history.json()["items"][0]["attachment_ids"] == [damaged["missing"]["id"]]
assert meta.status_code == 200
assert meta.json()["sha256"] == damaged["flipped"]["sha256"]
assert broken.status_code == 404
assert broken.json()["error"]["code"] == "not_found"
assert intact.status_code == 200
assert intact.content == SAMPLE
async def test_restored_object_leaves_quarantine(dsn: str) -> None:
async with stand(dsn) as s:
_, bob, _, _ = s.users
lost, flipped = await s.upload(), await s.upload()
del s.store.objects[key(lost)]
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
isolated = await s.run(apply=True, deep=True)
s.store.objects[key(lost)] = s.store.objects[key(flipped)] = (SAMPLE, "image/png")
s.store.modified[key(lost)] = s.store.modified[key(flipped)]
shallow = await s.run(apply=True)
again = await s.run(apply=True)
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
assert isolated.fixed["missing"] == isolated.fixed["sha_mismatch"] == 1
assert shallow.fixed["healed"] == 2
assert shallow.requests["s3_get"] == 2
assert not again.fixed
assert again.requests["s3_get"] == 0
assert hashlib.sha256(content.content).hexdigest() == lost["sha256"]
async def test_other_bytes_of_the_same_length_keep_the_quarantine(dsn: str) -> None:
async with stand(dsn) as s:
_, bob, _, _ = s.users
lost = await s.upload()
del s.store.objects[key(lost)]
isolated = await s.run(apply=True)
s.store.objects[key(lost)] = (bytes(len(SAMPLE)), "image/png")
s.age(key(lost))
shallow = await s.run(apply=True)
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
reason = await s.sql("SELECT quarantine_reason FROM attachments WHERE id = $1", lost["id"])
assert isolated.fixed["missing"] == 1
assert not shallow.fixed
assert shallow.found["quarantined"] == 1
assert shallow.requests["s3_get"] == 1
assert content.status_code == 404
assert reason[0]["quarantine_reason"] == "missing"
async def test_pages_of_rows_are_not_followed_by_an_empty_one(dsn: str) -> None:
async with stand(dsn) as s:
empty = await s.run()
for _ in range(4):
await s.upload()
full = await s.run()
assert (empty.rows, empty.requests["sql"]) == (0, 1)
assert (full.rows, full.requests["sql"]) == (4, 2)
def test_any_pass_failure_ends_as_reconcile_failed(
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
async def broken(*_args: Any) -> Reconciliation:
cancelled = TimeoutError("statement timeout")
raise ExceptionGroup("pass", [cancelled])
monkeypatch.setattr(entrypoint, "reconcile", broken)
for name, value in {
"DATABASE_URL": "postgresql://unused@127.0.0.1:1/unused",
"S3_ACCESS_KEY_ID": "reconciler",
"S3_SECRET_ACCESS_KEY": "reconciler-secret",
}.items():
monkeypatch.setenv(name, value)
with pytest.raises(SystemExit) as stopped:
entrypoint.main(["--apply"])
captured = capsys.readouterr()
assert stopped.value.code == 1
assert captured.out == ""
assert "reconcile.failed" in captured.err
assert "TimeoutError" in captured.err
assert "Traceback" not in captured.err
async def test_row_written_after_listing_keeps_the_object(dsn: str) -> None:
async with stand(dsn) as s:
first, last = await s.upload(), await s.upload()
fresh, late = f"{key(first)}-a", f"{key(first)}-b"
for name in (fresh, late):
s.store.objects[name] = (SAMPLE, "image/png")
s.store.modified[name] = datetime.now(UTC)
s.age(late)
async def row_appears() -> None:
s.store.after_page = None
await s.sql(LATE_ROW, first["id"], late)
s.store.after_page = row_appears
report = await s.run(apply=True)
objects = s.state()
assert key(first) < fresh < late < key(last)
assert report.found["orphan_stale"] == 1
assert report.rechecked["orphan_stale"] == 1
assert not report.fixed
assert late in objects
async def test_second_pass_is_skipped_while_one_runs(dsn: str) -> None:
async with stand(dsn) as s:
orphan = f"attachments/{uuid4()}"
s.store.objects[orphan] = (SAMPLE, "image/png")
s.age(orphan)
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
busy = await s.run(apply=True)
free = await s.run(apply=True)
assert held
assert busy.skipped
assert not busy.found
assert not free.skipped
assert free.fixed["orphan_stale"] == 1
async def test_storage_failure_mid_pass_ends_it_as_unavailable(dsn: str) -> None:
async with stand(dsn) as s:
await damage(s)
async def broken() -> None:
raise Unavailable("S3 недоступно") from TimeoutError()
s.store.after_page = broken
with pytest.raises(Unavailable) as failed:
await s.run(apply=True, deep=True)
assert isinstance(failed.value.__cause__, TimeoutError)
async def test_cancelled_pass_releases_its_autocommit_lock(dsn: str) -> None:
async with stand(dsn) as s:
entered = asyncio.Event()
async def hold() -> None:
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
assert held
entered.set()
await asyncio.Event().wait()
task = asyncio.create_task(hold())
try:
await asyncio.wait_for(entered.wait(), 2)
states = await s.sql(
"SELECT a.state FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid "
"WHERE l.locktype = 'advisory' AND l.objid = $1",
LOCK_KEY,
)
assert [row["state"] for row in states] == ["idle"]
finally:
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
locks = await s.sql(
"SELECT count(*) AS total FROM pg_locks WHERE locktype = 'advisory' AND objid = $1",
LOCK_KEY,
)
assert locks[0]["total"] == 0
async def test_legacy_reconciler_does_not_touch_pipeline_objects(dsn: str) -> None:
async with stand(dsn) as s:
alice, _, _, chat = s.users
response = await upload(s.http, alice, chat)
assert response.status_code == 202
await settle(s.http)
before = s.state()
assert {key.split("/", 1)[0] for key in before} == {"staging", "originals", "variants"}
for object_key in before:
s.age(object_key)
report = await s.run(apply=True, deep=True)
assert report.rows == report.objects == 0
assert not report.fixed and not report.errors
assert s.state() == before
content = await s.http.get(
f"/api/v1/attachments/{response.json()['id']}/content", headers=actor(alice)
)
assert content.status_code == 200 and content.content == SAMPLE