forked from mtusi2026/mtusi2026lab4
274 lines
10 KiB
Python
274 lines
10 KiB
Python
"""Real PostgreSQL + S3 fault injection for grade 5, track 2."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
from io import BytesIO
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import select, update
|
|
|
|
from app.config import load_settings
|
|
from app.db import create_engine, create_session_factory
|
|
from app.models.attachment import AttachmentRow, MessageAttachmentRow
|
|
from app.models.chat import ChatMemberRow, ChatRow
|
|
from app.models.message import MessageRow
|
|
from app.models.user import UserRow
|
|
from app.storage import S3Storage
|
|
from app.tools.reconcile_attachments import MISSING_ERROR_CODE, reconcile
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
not os.getenv('DATABASE_URL'), reason='requires the compose PostgreSQL and S3 services'
|
|
)
|
|
|
|
|
|
class AfterListStorage:
|
|
def __init__(self, storage, after_list):
|
|
self._storage = storage
|
|
self._after_list = after_list
|
|
|
|
async def list(self, prefix=''):
|
|
snapshot = await self._storage.list(prefix)
|
|
if self._after_list is not None:
|
|
after_list, self._after_list = self._after_list, None
|
|
await after_list()
|
|
return snapshot
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self._storage, name)
|
|
|
|
|
|
def test_dry_run_apply_and_repeat_are_safe():
|
|
"""Проверяет пробный, изменяющий и повторный запуски."""
|
|
asyncio.run(_check_dry_run_apply_and_repeat_are_safe())
|
|
|
|
|
|
async def _check_dry_run_apply_and_repeat_are_safe():
|
|
settings = load_settings()
|
|
engine = create_engine(settings.database_url)
|
|
sessions = create_session_factory(engine)
|
|
storage = S3Storage(
|
|
settings.s3_endpoint,
|
|
settings.s3_access_key_id,
|
|
settings.s3_secret_access_key,
|
|
settings.s3_bucket,
|
|
settings.s3_region,
|
|
)
|
|
now = datetime.now(timezone.utc)
|
|
old = now - timedelta(hours=2)
|
|
simulated_after_grace = now + timedelta(seconds=61)
|
|
user_id, chat_id, message_id = uuid4(), uuid4(), uuid4()
|
|
correct_id, missing_id, restored_id = uuid4(), uuid4(), uuid4()
|
|
changed_id, appeared_id = uuid4(), uuid4()
|
|
correct_key = f'originals/reconcile-correct-{uuid4()}'
|
|
orphan_key = f'originals/reconcile-orphan-{uuid4()}'
|
|
restored_key = f'originals/reconcile-restored-{uuid4()}'
|
|
changed_key = f'originals/reconcile-changed-{uuid4()}'
|
|
appeared_key = f'originals/reconcile-appeared-{uuid4()}'
|
|
fresh_key = f'originals/reconcile-fresh-{uuid4()}'
|
|
|
|
try:
|
|
async with sessions() as session:
|
|
async with session.begin():
|
|
session.add(UserRow(
|
|
id=user_id,
|
|
username=f'reconcile-{user_id.hex[:16]}',
|
|
display_name='Reconciler test',
|
|
password_hash='not-used',
|
|
created_at=old,
|
|
))
|
|
session.add(ChatRow(id=chat_id, title='Reconciler test', created_at=old))
|
|
await session.flush()
|
|
session.add(ChatMemberRow(chat_id=chat_id, user_id=user_id))
|
|
for attachment_id, key in (
|
|
(correct_id, correct_key),
|
|
(missing_id, f'originals/reconcile-missing-{uuid4()}'),
|
|
(restored_id, restored_key),
|
|
(changed_id, changed_key),
|
|
):
|
|
session.add(AttachmentRow(
|
|
id=attachment_id,
|
|
chat_id=chat_id,
|
|
owner_id=user_id,
|
|
filename='test.png',
|
|
object_key=key,
|
|
content_type='image/png',
|
|
size_bytes=1,
|
|
sha256='0' * 64,
|
|
width=1,
|
|
height=1,
|
|
status='ready',
|
|
error_code=None,
|
|
created_at=old,
|
|
updated_at=old,
|
|
))
|
|
await session.flush()
|
|
session.add(MessageRow(
|
|
id=message_id,
|
|
chat_id=chat_id,
|
|
sender_id=user_id,
|
|
text='used attachment must survive',
|
|
created_at=old,
|
|
idempotency_key=f'reconcile-{message_id}',
|
|
))
|
|
await session.flush()
|
|
session.add(MessageAttachmentRow(
|
|
message_id=message_id,
|
|
attachment_id=correct_id,
|
|
position=0,
|
|
))
|
|
|
|
for key in (correct_key, orphan_key):
|
|
await storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream')
|
|
|
|
dry_run = await reconcile(
|
|
sessions,
|
|
storage,
|
|
apply=False,
|
|
grace_seconds=60,
|
|
now=simulated_after_grace,
|
|
output=lambda _line: None,
|
|
)
|
|
assert dry_run['orphan'] == 1
|
|
assert dry_run['missing'] == 3
|
|
assert dry_run['removed'] == 0
|
|
assert dry_run['marked_failed'] == 0
|
|
assert {item.key for item in await storage.list('originals/')} >= {
|
|
correct_key,
|
|
orphan_key,
|
|
}
|
|
|
|
async def restore_missing_after_snapshot():
|
|
await storage.put(restored_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
|
async with sessions() as session:
|
|
async with session.begin():
|
|
await session.execute(
|
|
update(AttachmentRow)
|
|
.where(AttachmentRow.id == changed_id)
|
|
.values(updated_at=now)
|
|
)
|
|
|
|
applied = await reconcile(
|
|
sessions,
|
|
AfterListStorage(storage, restore_missing_after_snapshot),
|
|
apply=True,
|
|
grace_seconds=60,
|
|
now=simulated_after_grace,
|
|
output=lambda _line: None,
|
|
)
|
|
assert applied['removed'] == 1
|
|
assert applied['marked_failed'] == 1
|
|
assert applied['race_protected'] == 2
|
|
|
|
# Make the concurrently changed metadata healthy before checking that
|
|
# the next complete run has no work left.
|
|
await storage.put(changed_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
|
|
|
repeated = await reconcile(
|
|
sessions,
|
|
storage,
|
|
apply=True,
|
|
grace_seconds=60,
|
|
now=simulated_after_grace,
|
|
output=lambda _line: None,
|
|
)
|
|
assert repeated['orphan'] == 0
|
|
assert repeated['missing'] == 0
|
|
assert repeated['removed'] == 0
|
|
assert repeated['marked_failed'] == 0
|
|
|
|
object_keys = {item.key for item in await storage.list('originals/')}
|
|
assert correct_key in object_keys
|
|
assert restored_key in object_keys
|
|
assert changed_key in object_keys
|
|
assert orphan_key not in object_keys
|
|
async with sessions() as session:
|
|
missing_row = await session.get(AttachmentRow, missing_id)
|
|
assert missing_row is not None
|
|
assert missing_row.status == 'failed'
|
|
assert missing_row.error_code == MISSING_ERROR_CODE
|
|
assert await session.scalar(
|
|
select(MessageAttachmentRow.attachment_id).where(
|
|
MessageAttachmentRow.message_id == message_id
|
|
)
|
|
) == correct_id
|
|
|
|
restored_row = await session.get(AttachmentRow, restored_id)
|
|
assert restored_row is not None
|
|
assert restored_row.status == 'ready'
|
|
|
|
changed_row = await session.get(AttachmentRow, changed_id)
|
|
assert changed_row is not None
|
|
assert changed_row.status == 'ready'
|
|
|
|
# An object younger than the grace period survives a real apply run.
|
|
await storage.put(fresh_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
|
fresh_apply = await reconcile(
|
|
sessions,
|
|
storage,
|
|
apply=True,
|
|
grace_seconds=60,
|
|
now=datetime.now(timezone.utc),
|
|
output=lambda _line: None,
|
|
)
|
|
assert fresh_apply['fresh_protected'] == 1
|
|
assert fresh_apply['removed'] == 0
|
|
assert await storage.exists(fresh_key)
|
|
await storage.remove(fresh_key)
|
|
|
|
# Metadata created after the initial snapshots also protects an old
|
|
# object from the destructive orphan action.
|
|
await storage.put(appeared_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
|
|
|
async def create_metadata_after_snapshot():
|
|
async with sessions() as session:
|
|
async with session.begin():
|
|
session.add(AttachmentRow(
|
|
id=appeared_id,
|
|
chat_id=chat_id,
|
|
owner_id=user_id,
|
|
filename='appeared.png',
|
|
object_key=appeared_key,
|
|
content_type='image/png',
|
|
size_bytes=1,
|
|
sha256='0' * 64,
|
|
width=1,
|
|
height=1,
|
|
status='ready',
|
|
error_code=None,
|
|
created_at=old,
|
|
updated_at=old,
|
|
))
|
|
|
|
appeared_apply = await reconcile(
|
|
sessions,
|
|
AfterListStorage(storage, create_metadata_after_snapshot),
|
|
apply=True,
|
|
grace_seconds=60,
|
|
now=datetime.now(timezone.utc) + timedelta(seconds=61),
|
|
output=lambda _line: None,
|
|
)
|
|
assert appeared_apply['orphan'] == 1
|
|
assert appeared_apply['removed'] == 0
|
|
assert appeared_apply['race_protected'] == 1
|
|
assert await storage.exists(appeared_key)
|
|
finally:
|
|
await storage.remove(correct_key)
|
|
await storage.remove(orphan_key)
|
|
await storage.remove(restored_key)
|
|
await storage.remove(changed_key)
|
|
await storage.remove(appeared_key)
|
|
await storage.remove(fresh_key)
|
|
async with sessions() as session:
|
|
async with session.begin():
|
|
chat = await session.get(ChatRow, chat_id)
|
|
if chat is not None:
|
|
await session.delete(chat)
|
|
await session.flush()
|
|
user = await session.get(UserRow, user_id)
|
|
if user is not None:
|
|
await session.delete(user)
|
|
await engine.dispose()
|