Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files. Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
433 lines
16 KiB
Python
433 lines
16 KiB
Python
import asyncio
|
|
import hashlib
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from sqlalchemy import make_url
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from sqlalchemy.pool import NullPool
|
|
from test_attachments import storage, trio, upload
|
|
|
|
from conftest import actor, keyed, running_app, upgrade
|
|
from fakes import MemoryObjectStorage, NullLogger, RecordingPublisher
|
|
from messenger.attachments.application.commands import ReconcileStorage
|
|
from messenger.attachments.application.reconcile import Reconciliation, StorageReconciler
|
|
from messenger.attachments.domain.events import AttachmentDiscarded
|
|
from messenger.attachments.entrypoints import reconcile as entrypoint
|
|
from messenger.attachments.entrypoints.reconcile import LOCK_KEY
|
|
from messenger.attachments.infrastructure.postgres import (
|
|
PostgresAttachmentsUnitOfWork,
|
|
PostgresSingleRun,
|
|
)
|
|
from messenger.shared.application.errors import Unavailable
|
|
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
|
from messenger.shared.infrastructure.postgres import Database
|
|
from pictures import SAMPLE
|
|
from pipeline import settle
|
|
|
|
GRACE = timedelta(seconds=60)
|
|
OLD = timedelta(hours=2)
|
|
AGED_PENDING = """UPDATE attachments SET status = 'pending', staging_key = object_key,
|
|
original_key = NULL, updated_at = created_at,
|
|
created_at = created_at - interval '2 hours' WHERE id = $1"""
|
|
LATE_ROW = """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes,
|
|
sha256, width, height, object_key, staging_key, status, created_at, updated_at)
|
|
SELECT gen_random_uuid(), chat_id, owner_id, 'late.png', content_type, size_bytes, sha256,
|
|
width, height, $2, $2, 'pending', now(), now()
|
|
FROM attachments WHERE id = $1"""
|
|
|
|
|
|
@pytest.fixture
|
|
def dsn(database: str) -> str:
|
|
name = f"reconcile_{uuid4().hex[:12]}"
|
|
|
|
async def create() -> None:
|
|
engine = create_async_engine(
|
|
make_url(database).set(drivername="postgresql+asyncpg"),
|
|
isolation_level="AUTOCOMMIT",
|
|
poolclass=NullPool,
|
|
)
|
|
try:
|
|
async with engine.connect() as connection:
|
|
await connection.exec_driver_sql(f"CREATE DATABASE {name}")
|
|
finally:
|
|
await engine.dispose()
|
|
await upgrade(target)
|
|
|
|
target = f"{database.rsplit('/', 1)[0]}/{name}"
|
|
asyncio.run(create())
|
|
return target
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Stand:
|
|
http: AsyncClient
|
|
store: MemoryObjectStorage
|
|
database: Database
|
|
users: tuple[dict[str, Any], ...]
|
|
published: RecordingPublisher
|
|
|
|
async def run(self, *, apply: bool = False, deep: bool = False) -> Reconciliation:
|
|
reconciler = StorageReconciler(
|
|
lambda: PostgresAttachmentsUnitOfWork(self.database),
|
|
self.store,
|
|
PostgresSingleRun(self.database, LOCK_KEY),
|
|
MonotonicUtcClock(),
|
|
logger=NullLogger(),
|
|
publisher=self.published,
|
|
)
|
|
command = ReconcileStorage(
|
|
apply=apply, grace=GRACE, skew=timedelta(0), deep=deep, page_size=2
|
|
)
|
|
return await reconciler(command)
|
|
|
|
async def sql(self, query: str, *args: Any) -> Any:
|
|
async with self.database.session() as session:
|
|
connection = await session.connection()
|
|
result = await connection.exec_driver_sql(query, args)
|
|
rows = result.mappings().all() if result.returns_rows else []
|
|
await session.commit()
|
|
return rows
|
|
|
|
async def upload(self) -> dict[str, Any]:
|
|
alice, _, _, chat = self.users
|
|
response = await upload(self.http, alice, chat)
|
|
assert response.status_code == 202, response.text
|
|
attachment = response.json()
|
|
legacy_key = key(attachment)
|
|
staging_key = f"staging/{attachment['id']}"
|
|
self.store.objects[legacy_key] = self.store.objects.pop(staging_key)
|
|
self.store.modified[legacy_key] = self.store.modified.pop(staging_key)
|
|
await self.sql(
|
|
"DELETE FROM outbox WHERE job_id IN (SELECT job_id FROM jobs WHERE attachment_id = $1)",
|
|
attachment["id"],
|
|
)
|
|
await self.sql("DELETE FROM jobs WHERE attachment_id = $1", attachment["id"])
|
|
await self.sql(
|
|
"UPDATE attachments SET status = 'ready', object_key = $2, original_key = $2, "
|
|
"staging_key = NULL, published_version = 1 WHERE id = $1",
|
|
attachment["id"],
|
|
legacy_key,
|
|
)
|
|
return attachment
|
|
|
|
def age(self, key: str) -> None:
|
|
self.store.modified[key] = datetime.now(UTC) - OLD
|
|
|
|
def state(self) -> dict[str, bytes]:
|
|
return {key: data for key, (data, _) in self.store.objects.items()}
|
|
|
|
|
|
@asynccontextmanager
|
|
async def stand(dsn: str) -> AsyncIterator[Stand]:
|
|
async with running_app(database_url=dsn) as http:
|
|
database = Database(
|
|
dsn, max_size=12, connect_timeout=2, acquire_timeout=2, command_timeout=5
|
|
)
|
|
await database.open()
|
|
try:
|
|
yield Stand(http, await storage(http), database, await trio(http), RecordingPublisher())
|
|
finally:
|
|
await database.close()
|
|
|
|
|
|
def key(attachment: dict[str, Any]) -> str:
|
|
return f"attachments/{attachment['id']}"
|
|
|
|
|
|
async def damage(s: Stand) -> dict[str, Any]:
|
|
alice, _, _, chat = s.users
|
|
healthy, missing, sized, flipped, stale, fresh = [await s.upload() for _ in range(6)]
|
|
posted = await s.http.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [missing["id"]]},
|
|
headers=keyed(alice),
|
|
)
|
|
assert posted.status_code == 201, posted.text
|
|
del s.store.objects[key(missing)]
|
|
s.store.objects[key(sized)] = (SAMPLE[:100], "image/png")
|
|
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
|
|
await s.sql(AGED_PENDING, stale["id"])
|
|
await s.sql(
|
|
"UPDATE attachments SET status = 'pending', staging_key = object_key, "
|
|
"original_key = NULL WHERE id = $1",
|
|
fresh["id"],
|
|
)
|
|
orphan, newborn = f"attachments/{uuid4()}", f"attachments/{uuid4()}"
|
|
s.store.objects[orphan] = s.store.objects[newborn] = (SAMPLE, "image/png")
|
|
s.store.modified[newborn] = datetime.now(UTC)
|
|
s.age(orphan)
|
|
return {
|
|
"healthy": healthy,
|
|
"missing": missing,
|
|
"sized": sized,
|
|
"flipped": flipped,
|
|
"stale": stale,
|
|
"fresh": fresh,
|
|
"orphan": orphan,
|
|
"newborn": newborn,
|
|
"message": posted.json(),
|
|
}
|
|
|
|
|
|
EVERY_CLASS = {
|
|
"orphan_stale": 1,
|
|
"orphan_fresh": 1,
|
|
"missing": 1,
|
|
"size_mismatch": 1,
|
|
"sha_mismatch": 1,
|
|
"pending_stale": 1,
|
|
"pending_fresh": 1,
|
|
}
|
|
|
|
|
|
async def test_dry_run_reports_every_class_and_changes_nothing(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
await damage(s)
|
|
objects = s.state()
|
|
rows = await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id")
|
|
shallow = await s.run()
|
|
deep = await s.run(deep=True)
|
|
assert s.state() == objects
|
|
assert (
|
|
await s.sql("SELECT id, status, quarantine_reason FROM attachments ORDER BY id") == rows
|
|
)
|
|
assert dict(deep.found) == EVERY_CLASS
|
|
assert dict(shallow.found) == {k: v for k, v in EVERY_CLASS.items() if k != "sha_mismatch"}
|
|
assert not deep.fixed
|
|
assert not shallow.fixed
|
|
assert shallow.requests == {"s3_list": 4, "sql": 3}
|
|
assert deep.requests["s3_get"] == 2
|
|
|
|
|
|
async def test_apply_fixes_what_it_found_and_repeats_as_noop(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
damaged = await damage(s)
|
|
alice, bob, _, chat = s.users
|
|
first = await s.run(apply=True, deep=True)
|
|
second = await s.run(apply=True, deep=True)
|
|
reasons = {
|
|
str(row["id"]): row["quarantine_reason"]
|
|
for row in await s.sql("SELECT id, quarantine_reason FROM attachments")
|
|
}
|
|
history = await s.http.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(bob))
|
|
meta = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['flipped']['id']}", headers=actor(bob)
|
|
)
|
|
broken = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['flipped']['id']}/content", headers=actor(bob)
|
|
)
|
|
intact = await s.http.get(
|
|
f"/api/v1/attachments/{damaged['healthy']['id']}/content", headers=actor(alice)
|
|
)
|
|
objects = s.state()
|
|
assert {name: first.fixed[name] for name in EVERY_CLASS} == {
|
|
**EVERY_CLASS,
|
|
"orphan_fresh": 0,
|
|
"pending_fresh": 0,
|
|
}
|
|
assert not second.fixed
|
|
assert dict(second.found) == {"orphan_fresh": 1, "pending_fresh": 1, "quarantined": 3}
|
|
assert reasons[damaged["missing"]["id"]] == "missing"
|
|
assert reasons[damaged["sized"]["id"]] == "size_mismatch"
|
|
assert reasons[damaged["flipped"]["id"]] == "sha_mismatch"
|
|
assert reasons[damaged["healthy"]["id"]] is None
|
|
assert [
|
|
(event.attachment_id, event.reason)
|
|
for event in s.published.events
|
|
if isinstance(event, AttachmentDiscarded)
|
|
] == [(UUID(damaged["stale"]["id"]), "reconcile")]
|
|
assert reasons[damaged["fresh"]["id"]] is None
|
|
assert damaged["stale"]["id"] not in reasons
|
|
assert key(damaged["stale"]) not in objects
|
|
assert damaged["orphan"] not in objects
|
|
assert damaged["newborn"] in objects
|
|
assert key(damaged["flipped"]) in objects
|
|
assert history.json()["items"][0]["attachment_ids"] == [damaged["missing"]["id"]]
|
|
assert meta.status_code == 200
|
|
assert meta.json()["sha256"] == damaged["flipped"]["sha256"]
|
|
assert broken.status_code == 404
|
|
assert broken.json()["error"]["code"] == "not_found"
|
|
assert intact.status_code == 200
|
|
assert intact.content == SAMPLE
|
|
|
|
|
|
async def test_restored_object_leaves_quarantine(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
_, bob, _, _ = s.users
|
|
lost, flipped = await s.upload(), await s.upload()
|
|
del s.store.objects[key(lost)]
|
|
s.store.objects[key(flipped)] = (SAMPLE[::-1], "image/png")
|
|
isolated = await s.run(apply=True, deep=True)
|
|
s.store.objects[key(lost)] = s.store.objects[key(flipped)] = (SAMPLE, "image/png")
|
|
s.store.modified[key(lost)] = s.store.modified[key(flipped)]
|
|
shallow = await s.run(apply=True)
|
|
again = await s.run(apply=True)
|
|
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
|
|
assert isolated.fixed["missing"] == isolated.fixed["sha_mismatch"] == 1
|
|
assert shallow.fixed["healed"] == 2
|
|
assert shallow.requests["s3_get"] == 2
|
|
assert not again.fixed
|
|
assert again.requests["s3_get"] == 0
|
|
assert hashlib.sha256(content.content).hexdigest() == lost["sha256"]
|
|
|
|
|
|
async def test_other_bytes_of_the_same_length_keep_the_quarantine(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
_, bob, _, _ = s.users
|
|
lost = await s.upload()
|
|
del s.store.objects[key(lost)]
|
|
isolated = await s.run(apply=True)
|
|
s.store.objects[key(lost)] = (bytes(len(SAMPLE)), "image/png")
|
|
s.age(key(lost))
|
|
shallow = await s.run(apply=True)
|
|
content = await s.http.get(f"/api/v1/attachments/{lost['id']}/content", headers=actor(bob))
|
|
reason = await s.sql("SELECT quarantine_reason FROM attachments WHERE id = $1", lost["id"])
|
|
assert isolated.fixed["missing"] == 1
|
|
assert not shallow.fixed
|
|
assert shallow.found["quarantined"] == 1
|
|
assert shallow.requests["s3_get"] == 1
|
|
assert content.status_code == 404
|
|
assert reason[0]["quarantine_reason"] == "missing"
|
|
|
|
|
|
async def test_pages_of_rows_are_not_followed_by_an_empty_one(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
empty = await s.run()
|
|
for _ in range(4):
|
|
await s.upload()
|
|
full = await s.run()
|
|
assert (empty.rows, empty.requests["sql"]) == (0, 1)
|
|
assert (full.rows, full.requests["sql"]) == (4, 2)
|
|
|
|
|
|
def test_any_pass_failure_ends_as_reconcile_failed(
|
|
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
async def broken(*_args: Any) -> Reconciliation:
|
|
cancelled = TimeoutError("statement timeout")
|
|
raise ExceptionGroup("pass", [cancelled])
|
|
|
|
monkeypatch.setattr(entrypoint, "reconcile", broken)
|
|
for name, value in {
|
|
"DATABASE_URL": "postgresql://unused@127.0.0.1:1/unused",
|
|
"S3_ACCESS_KEY_ID": "reconciler",
|
|
"S3_SECRET_ACCESS_KEY": "reconciler-secret",
|
|
}.items():
|
|
monkeypatch.setenv(name, value)
|
|
with pytest.raises(SystemExit) as stopped:
|
|
entrypoint.main(["--apply"])
|
|
captured = capsys.readouterr()
|
|
assert stopped.value.code == 1
|
|
assert captured.out == ""
|
|
assert "reconcile.failed" in captured.err
|
|
assert "TimeoutError" in captured.err
|
|
assert "Traceback" not in captured.err
|
|
|
|
|
|
async def test_row_written_after_listing_keeps_the_object(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
first, last = await s.upload(), await s.upload()
|
|
fresh, late = f"{key(first)}-a", f"{key(first)}-b"
|
|
for name in (fresh, late):
|
|
s.store.objects[name] = (SAMPLE, "image/png")
|
|
s.store.modified[name] = datetime.now(UTC)
|
|
s.age(late)
|
|
|
|
async def row_appears() -> None:
|
|
s.store.after_page = None
|
|
await s.sql(LATE_ROW, first["id"], late)
|
|
|
|
s.store.after_page = row_appears
|
|
report = await s.run(apply=True)
|
|
objects = s.state()
|
|
assert key(first) < fresh < late < key(last)
|
|
assert report.found["orphan_stale"] == 1
|
|
assert report.rechecked["orphan_stale"] == 1
|
|
assert not report.fixed
|
|
assert late in objects
|
|
|
|
|
|
async def test_second_pass_is_skipped_while_one_runs(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
orphan = f"attachments/{uuid4()}"
|
|
s.store.objects[orphan] = (SAMPLE, "image/png")
|
|
s.age(orphan)
|
|
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
|
|
busy = await s.run(apply=True)
|
|
free = await s.run(apply=True)
|
|
assert held
|
|
assert busy.skipped
|
|
assert not busy.found
|
|
assert not free.skipped
|
|
assert free.fixed["orphan_stale"] == 1
|
|
|
|
|
|
async def test_storage_failure_mid_pass_ends_it_as_unavailable(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
await damage(s)
|
|
|
|
async def broken() -> None:
|
|
raise Unavailable("S3 недоступно") from TimeoutError()
|
|
|
|
s.store.after_page = broken
|
|
with pytest.raises(Unavailable) as failed:
|
|
await s.run(apply=True, deep=True)
|
|
assert isinstance(failed.value.__cause__, TimeoutError)
|
|
|
|
|
|
async def test_cancelled_pass_releases_its_autocommit_lock(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
entered = asyncio.Event()
|
|
|
|
async def hold() -> None:
|
|
async with PostgresSingleRun(s.database, LOCK_KEY).hold() as held:
|
|
assert held
|
|
entered.set()
|
|
await asyncio.Event().wait()
|
|
|
|
task = asyncio.create_task(hold())
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 2)
|
|
states = await s.sql(
|
|
"SELECT a.state FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid "
|
|
"WHERE l.locktype = 'advisory' AND l.objid = $1",
|
|
LOCK_KEY,
|
|
)
|
|
assert [row["state"] for row in states] == ["idle"]
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
locks = await s.sql(
|
|
"SELECT count(*) AS total FROM pg_locks WHERE locktype = 'advisory' AND objid = $1",
|
|
LOCK_KEY,
|
|
)
|
|
assert locks[0]["total"] == 0
|
|
|
|
|
|
async def test_legacy_reconciler_does_not_touch_pipeline_objects(dsn: str) -> None:
|
|
async with stand(dsn) as s:
|
|
alice, _, _, chat = s.users
|
|
response = await upload(s.http, alice, chat)
|
|
assert response.status_code == 202
|
|
await settle(s.http)
|
|
before = s.state()
|
|
assert {key.split("/", 1)[0] for key in before} == {"staging", "originals", "variants"}
|
|
for object_key in before:
|
|
s.age(object_key)
|
|
report = await s.run(apply=True, deep=True)
|
|
assert report.rows == report.objects == 0
|
|
assert not report.fixed and not report.errors
|
|
assert s.state() == before
|
|
content = await s.http.get(
|
|
f"/api/v1/attachments/{response.json()['id']}/content", headers=actor(alice)
|
|
)
|
|
assert content.status_code == 200 and content.content == SAMPLE
|