Files
mtusi2026lab4/tests/test_reconciler_integration.py
T

274 lines
10 KiB
Python

"""Real PostgreSQL + S3 fault injection for grade 5, track 2."""
from __future__ import annotations
import asyncio
import os
from datetime import datetime, timedelta, timezone
from io import BytesIO
from uuid import uuid4
import pytest
from sqlalchemy import select, update
from app.config import load_settings
from app.db import create_engine, create_session_factory
from app.models.attachment import AttachmentRow, MessageAttachmentRow
from app.models.chat import ChatMemberRow, ChatRow
from app.models.message import MessageRow
from app.models.user import UserRow
from app.storage import S3Storage
from app.tools.reconcile_attachments import MISSING_ERROR_CODE, reconcile
pytestmark = pytest.mark.skipif(
not os.getenv('DATABASE_URL'), reason='requires the compose PostgreSQL and S3 services'
)
class AfterListStorage:
def __init__(self, storage, after_list):
self._storage = storage
self._after_list = after_list
async def list(self, prefix=''):
snapshot = await self._storage.list(prefix)
if self._after_list is not None:
after_list, self._after_list = self._after_list, None
await after_list()
return snapshot
def __getattr__(self, name):
return getattr(self._storage, name)
def test_dry_run_apply_and_repeat_are_safe():
"""Проверяет пробный, изменяющий и повторный запуски."""
asyncio.run(_check_dry_run_apply_and_repeat_are_safe())
async def _check_dry_run_apply_and_repeat_are_safe():
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
now = datetime.now(timezone.utc)
old = now - timedelta(hours=2)
simulated_after_grace = now + timedelta(seconds=61)
user_id, chat_id, message_id = uuid4(), uuid4(), uuid4()
correct_id, missing_id, restored_id = uuid4(), uuid4(), uuid4()
changed_id, appeared_id = uuid4(), uuid4()
correct_key = f'originals/reconcile-correct-{uuid4()}'
orphan_key = f'originals/reconcile-orphan-{uuid4()}'
restored_key = f'originals/reconcile-restored-{uuid4()}'
changed_key = f'originals/reconcile-changed-{uuid4()}'
appeared_key = f'originals/reconcile-appeared-{uuid4()}'
fresh_key = f'originals/reconcile-fresh-{uuid4()}'
try:
async with sessions() as session:
async with session.begin():
session.add(UserRow(
id=user_id,
username=f'reconcile-{user_id.hex[:16]}',
display_name='Reconciler test',
password_hash='not-used',
created_at=old,
))
session.add(ChatRow(id=chat_id, title='Reconciler test', created_at=old))
await session.flush()
session.add(ChatMemberRow(chat_id=chat_id, user_id=user_id))
for attachment_id, key in (
(correct_id, correct_key),
(missing_id, f'originals/reconcile-missing-{uuid4()}'),
(restored_id, restored_key),
(changed_id, changed_key),
):
session.add(AttachmentRow(
id=attachment_id,
chat_id=chat_id,
owner_id=user_id,
filename='test.png',
object_key=key,
content_type='image/png',
size_bytes=1,
sha256='0' * 64,
width=1,
height=1,
status='ready',
error_code=None,
created_at=old,
updated_at=old,
))
await session.flush()
session.add(MessageRow(
id=message_id,
chat_id=chat_id,
sender_id=user_id,
text='used attachment must survive',
created_at=old,
idempotency_key=f'reconcile-{message_id}',
))
await session.flush()
session.add(MessageAttachmentRow(
message_id=message_id,
attachment_id=correct_id,
position=0,
))
for key in (correct_key, orphan_key):
await storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream')
dry_run = await reconcile(
sessions,
storage,
apply=False,
grace_seconds=60,
now=simulated_after_grace,
output=lambda _line: None,
)
assert dry_run['orphan'] == 1
assert dry_run['missing'] == 3
assert dry_run['removed'] == 0
assert dry_run['marked_failed'] == 0
assert {item.key for item in await storage.list('originals/')} >= {
correct_key,
orphan_key,
}
async def restore_missing_after_snapshot():
await storage.put(restored_key, BytesIO(b'x'), 1, 'application/octet-stream')
async with sessions() as session:
async with session.begin():
await session.execute(
update(AttachmentRow)
.where(AttachmentRow.id == changed_id)
.values(updated_at=now)
)
applied = await reconcile(
sessions,
AfterListStorage(storage, restore_missing_after_snapshot),
apply=True,
grace_seconds=60,
now=simulated_after_grace,
output=lambda _line: None,
)
assert applied['removed'] == 1
assert applied['marked_failed'] == 1
assert applied['race_protected'] == 2
# Make the concurrently changed metadata healthy before checking that
# the next complete run has no work left.
await storage.put(changed_key, BytesIO(b'x'), 1, 'application/octet-stream')
repeated = await reconcile(
sessions,
storage,
apply=True,
grace_seconds=60,
now=simulated_after_grace,
output=lambda _line: None,
)
assert repeated['orphan'] == 0
assert repeated['missing'] == 0
assert repeated['removed'] == 0
assert repeated['marked_failed'] == 0
object_keys = {item.key for item in await storage.list('originals/')}
assert correct_key in object_keys
assert restored_key in object_keys
assert changed_key in object_keys
assert orphan_key not in object_keys
async with sessions() as session:
missing_row = await session.get(AttachmentRow, missing_id)
assert missing_row is not None
assert missing_row.status == 'failed'
assert missing_row.error_code == MISSING_ERROR_CODE
assert await session.scalar(
select(MessageAttachmentRow.attachment_id).where(
MessageAttachmentRow.message_id == message_id
)
) == correct_id
restored_row = await session.get(AttachmentRow, restored_id)
assert restored_row is not None
assert restored_row.status == 'ready'
changed_row = await session.get(AttachmentRow, changed_id)
assert changed_row is not None
assert changed_row.status == 'ready'
# An object younger than the grace period survives a real apply run.
await storage.put(fresh_key, BytesIO(b'x'), 1, 'application/octet-stream')
fresh_apply = await reconcile(
sessions,
storage,
apply=True,
grace_seconds=60,
now=datetime.now(timezone.utc),
output=lambda _line: None,
)
assert fresh_apply['fresh_protected'] == 1
assert fresh_apply['removed'] == 0
assert await storage.exists(fresh_key)
await storage.remove(fresh_key)
# Metadata created after the initial snapshots also protects an old
# object from the destructive orphan action.
await storage.put(appeared_key, BytesIO(b'x'), 1, 'application/octet-stream')
async def create_metadata_after_snapshot():
async with sessions() as session:
async with session.begin():
session.add(AttachmentRow(
id=appeared_id,
chat_id=chat_id,
owner_id=user_id,
filename='appeared.png',
object_key=appeared_key,
content_type='image/png',
size_bytes=1,
sha256='0' * 64,
width=1,
height=1,
status='ready',
error_code=None,
created_at=old,
updated_at=old,
))
appeared_apply = await reconcile(
sessions,
AfterListStorage(storage, create_metadata_after_snapshot),
apply=True,
grace_seconds=60,
now=datetime.now(timezone.utc) + timedelta(seconds=61),
output=lambda _line: None,
)
assert appeared_apply['orphan'] == 1
assert appeared_apply['removed'] == 0
assert appeared_apply['race_protected'] == 1
assert await storage.exists(appeared_key)
finally:
await storage.remove(correct_key)
await storage.remove(orphan_key)
await storage.remove(restored_key)
await storage.remove(changed_key)
await storage.remove(appeared_key)
await storage.remove(fresh_key)
async with sessions() as session:
async with session.begin():
chat = await session.get(ChatRow, chat_id)
if chat is not None:
await session.delete(chat)
await session.flush()
user = await session.get(UserRow, user_id)
if user is not None:
await session.delete(user)
await engine.dispose()