Carry sourcef591fa64e5forward from72a1e57. Preserve lab4 attachment transactions, reconciliation locks, quarantine, and existing schema data through verified legacy adoption. Keep report and evidence work in progress untouched.
449 lines
17 KiB
Python
449 lines
17 KiB
Python
import asyncio
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import sys
|
|
from collections.abc import AsyncIterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from alembic import command
|
|
from alembic.autogenerate import compare_metadata
|
|
from alembic.migration import MigrationContext
|
|
from alembic.script import ScriptDirectory
|
|
from alembic.util import CommandError
|
|
from sqlalchemy import MetaData, Table, func, insert, inspect, literal, make_url, select, text
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from sqlalchemy.pool import NullPool
|
|
|
|
from conftest import DATABASE, ROOT, migration_config, sql_connection, upgrade
|
|
from messenger.attachments.infrastructure.postgres import AttachmentRow
|
|
from messenger.identity.infrastructure.postgres import SessionRow, UserRow
|
|
from messenger.messaging.infrastructure.postgres import (
|
|
ChatMemberRow,
|
|
ChatRow,
|
|
IdempotencyKeyRow,
|
|
MessageAttachmentRow,
|
|
MessageRow,
|
|
)
|
|
from messenger.shared.infrastructure.postgres import Base
|
|
|
|
VERSIONS = ["0001_init", "0002_idempotency_keys", "0003_attachments", "0004_attachment_quarantine"]
|
|
TABLES = [
|
|
UserRow,
|
|
SessionRow,
|
|
ChatRow,
|
|
ChatMemberRow,
|
|
MessageRow,
|
|
IdempotencyKeyRow,
|
|
AttachmentRow,
|
|
MessageAttachmentRow,
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
async def empty_database(database: str) -> AsyncIterator[str]:
|
|
name = f"migration_test_{uuid4().hex}"
|
|
url = make_url(database).set(drivername="postgresql+asyncpg")
|
|
engine = create_async_engine(url, isolation_level="AUTOCOMMIT", poolclass=NullPool)
|
|
try:
|
|
async with engine.connect() as connection:
|
|
await connection.exec_driver_sql(f"CREATE DATABASE {name}")
|
|
yield url.set(database=name).render_as_string(hide_password=False)
|
|
finally:
|
|
async with engine.connect() as connection:
|
|
await connection.exec_driver_sql(f"DROP DATABASE {name} WITH (FORCE)")
|
|
await engine.dispose()
|
|
|
|
|
|
async def legacy(dsn: str, versions: int = 4) -> None:
|
|
async with sql_connection(dsn) as connection:
|
|
await connection.execute(
|
|
text(
|
|
"CREATE TABLE schema_migrations (version text PRIMARY KEY, checksum text NOT NULL, "
|
|
"applied_at timestamptz NOT NULL DEFAULT now())"
|
|
)
|
|
)
|
|
journal = await connection.run_sync(
|
|
lambda sync: Table("schema_migrations", MetaData(), autoload_with=sync)
|
|
)
|
|
for version in VERSIONS[:versions]:
|
|
sql = (ROOT / "student-tests" / "fixtures" / "legacy" / f"{version}.sql").read_text()
|
|
for statement in sql.split(";"):
|
|
if statement.strip():
|
|
await connection.execute(text(statement))
|
|
await connection.execute(
|
|
insert(journal).values(
|
|
version=version, checksum=hashlib.sha256(sql.encode()).hexdigest()
|
|
)
|
|
)
|
|
|
|
|
|
async def seed(dsn: str, versions: int = 4) -> None:
|
|
user, chat, message = uuid4(), uuid4(), uuid4()
|
|
stamp = datetime.now(UTC)
|
|
rows = [
|
|
(
|
|
UserRow,
|
|
{
|
|
"id": user,
|
|
"username": "legacy_user",
|
|
"display_name": "Старые данные",
|
|
"password_hash": "$argon2id$legacy",
|
|
"created_at": stamp,
|
|
},
|
|
),
|
|
(
|
|
SessionRow,
|
|
{
|
|
"token_hash": b"s" * 32,
|
|
"user_id": user,
|
|
"created_at": stamp,
|
|
"expires_at": stamp + timedelta(days=1),
|
|
},
|
|
),
|
|
(
|
|
ChatRow,
|
|
{
|
|
"id": chat,
|
|
"title": "Существующий чат",
|
|
"created_at": stamp,
|
|
"last_message_at": stamp,
|
|
},
|
|
),
|
|
(
|
|
ChatMemberRow,
|
|
{"chat_id": chat, "user_id": user, "position": 1, "chat_created_at": stamp},
|
|
),
|
|
(
|
|
MessageRow,
|
|
{
|
|
"id": message,
|
|
"chat_id": chat,
|
|
"sender_id": user,
|
|
"text": "Сохранить",
|
|
"created_at": stamp,
|
|
},
|
|
),
|
|
(
|
|
IdempotencyKeyRow,
|
|
{
|
|
"chat_id": chat,
|
|
"user_id": user,
|
|
"key": "legacy-key",
|
|
"fingerprint": b"f" * 32,
|
|
"message_id": message,
|
|
"created_at": stamp,
|
|
},
|
|
),
|
|
]
|
|
async with sql_connection(dsn) as connection:
|
|
for model, row in rows[: 5 if versions == 1 else 6]:
|
|
await connection.execute(insert(model).values(**row))
|
|
if versions >= 3:
|
|
attachment = uuid4()
|
|
row = {
|
|
"id": attachment,
|
|
"chat_id": chat,
|
|
"owner_id": user,
|
|
"filename": "legacy.png",
|
|
"content_type": "image/png",
|
|
"size_bytes": 17,
|
|
"sha256": "a" * 64,
|
|
"width": 1,
|
|
"height": 1,
|
|
"object_key": f"attachments/{attachment}",
|
|
"status": "ready",
|
|
"created_at": stamp,
|
|
"updated_at": stamp,
|
|
}
|
|
if versions >= 4:
|
|
row.update(quarantine_reason="missing", quarantined_at=stamp)
|
|
table = await connection.run_sync(
|
|
lambda sync: Table("attachments", MetaData(), autoload_with=sync)
|
|
)
|
|
await connection.execute(insert(table).values(**row))
|
|
await connection.execute(
|
|
insert(MessageAttachmentRow).values(
|
|
message_id=message, position=0, chat_id=chat, attachment_id=attachment
|
|
)
|
|
)
|
|
|
|
|
|
async def contents(dsn: str) -> dict:
|
|
async with sql_connection(dsn) as connection:
|
|
tables = await connection.run_sync(lambda sync: inspect(sync).get_table_names())
|
|
result = {}
|
|
for model in TABLES:
|
|
if model.__tablename__ not in tables:
|
|
continue
|
|
table = await connection.run_sync(
|
|
lambda sync, name=model.__tablename__: Table(name, MetaData(), autoload_with=sync)
|
|
)
|
|
columns = [
|
|
table.c[col.name] if col.name in table.c else literal(None).label(col.name)
|
|
for col in model.__table__.columns
|
|
]
|
|
result[model.__tablename__] = (
|
|
await connection.execute(select(*columns).order_by(*table.primary_key))
|
|
).all()
|
|
return result
|
|
|
|
|
|
async def revision(dsn: str) -> str | None:
|
|
async with sql_connection(dsn) as connection:
|
|
return await connection.run_sync(
|
|
lambda sync: MigrationContext.configure(sync).get_current_revision()
|
|
)
|
|
|
|
|
|
async def test_migrations_are_ordered_and_versioned() -> None:
|
|
scripts = ScriptDirectory.from_config(migration_config(DATABASE["url"]))
|
|
assert [script.revision for script in reversed(list(scripts.walk_revisions()))] == VERSIONS
|
|
assert await revision(DATABASE["url"]) == VERSIONS[-1]
|
|
|
|
|
|
async def test_fresh_schema_matches_models(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
assert (
|
|
await connection.run_sync(
|
|
lambda sync: compare_metadata(MigrationContext.configure(sync), Base.metadata)
|
|
)
|
|
== []
|
|
)
|
|
await asyncio.to_thread(command.check, migration_config(empty_database))
|
|
|
|
|
|
async def test_rerun_changes_nothing(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
await upgrade(empty_database)
|
|
assert await contents(empty_database) == before
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
|
|
|
|
@pytest.mark.parametrize("versions", [1, 2, 3, 4])
|
|
async def test_legacy_upgrade_preserves_every_row(empty_database: str, versions: int) -> None:
|
|
await legacy(empty_database, versions)
|
|
await seed(empty_database, versions)
|
|
before = await contents(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
journal = (
|
|
await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version"))
|
|
).all()
|
|
await upgrade(empty_database)
|
|
after = await contents(empty_database)
|
|
assert {name: after[name] for name in before} == before
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
async with sql_connection(empty_database) as connection:
|
|
assert (
|
|
await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version"))
|
|
).all() == journal
|
|
assert (
|
|
await connection.scalar(
|
|
text("SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'alembic_verify_%'")
|
|
)
|
|
== 0
|
|
)
|
|
await upgrade(empty_database)
|
|
assert await contents(empty_database) == after
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"damage",
|
|
[
|
|
"DROP INDEX messages_history_idx",
|
|
"ALTER TABLE messages DROP CONSTRAINT messages_chat_id_sender_id_fkey",
|
|
"ALTER TABLE users DROP CONSTRAINT users_username_format",
|
|
"ALTER TABLE users ALTER COLUMN display_name TYPE varchar",
|
|
'ALTER TABLE users ALTER COLUMN username TYPE text COLLATE "C"',
|
|
"ALTER TABLE sessions SET UNLOGGED",
|
|
"ALTER TABLE sessions ALTER COLUMN expires_at DROP NOT NULL",
|
|
"UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0001_init'",
|
|
"DELETE FROM schema_migrations WHERE version = '0001_init'",
|
|
"UPDATE schema_migrations SET version = 'unknown' WHERE version = '0002_idempotency_keys'",
|
|
"DROP TABLE schema_migrations",
|
|
"ALTER TABLE attachments DROP CONSTRAINT attachments_owner_fkey",
|
|
"ALTER TABLE attachments DROP CONSTRAINT attachments_quarantine_check",
|
|
"ALTER TABLE message_attachments DROP CONSTRAINT message_attachments_attachment_fkey",
|
|
'ALTER TABLE attachments ALTER COLUMN object_key TYPE text COLLATE "default"',
|
|
"DROP INDEX attachments_pending_idx",
|
|
"UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0003_attachments'",
|
|
],
|
|
)
|
|
async def test_legacy_drift_is_refused_without_stamping(empty_database: str, damage: str) -> None:
|
|
await legacy(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
await connection.execute(text(damage))
|
|
with pytest.raises(RuntimeError, match=r"[Ll]egacy|[Ee]xisting|Unknown"):
|
|
await upgrade(empty_database)
|
|
assert await revision(empty_database) is None
|
|
assert await contents(empty_database) == before
|
|
|
|
|
|
async def test_unknown_revision_is_refused(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
await connection.execute(text("UPDATE alembic_version SET version_num = 'unknown'"))
|
|
with pytest.raises(CommandError, match="unknown"):
|
|
await upgrade(empty_database)
|
|
assert await contents(empty_database) == before
|
|
assert await revision(empty_database) == "unknown"
|
|
|
|
|
|
@pytest.mark.parametrize("existing", [False, True])
|
|
async def test_simultaneous_migration_processes_serialize(
|
|
empty_database: str, existing: bool
|
|
) -> None:
|
|
if existing:
|
|
await legacy(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
processes = []
|
|
try:
|
|
for _ in range(2):
|
|
processes.append(
|
|
await asyncio.create_subprocess_exec(
|
|
sys.executable,
|
|
"-m",
|
|
"alembic",
|
|
"upgrade",
|
|
"head",
|
|
cwd=ROOT,
|
|
env={**os.environ, "DATABASE_URL": empty_database},
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
)
|
|
async with asyncio.timeout(30):
|
|
output = await asyncio.gather(*(process.communicate() for process in processes))
|
|
assert [process.returncode for process in processes] == [0, 0], output
|
|
finally:
|
|
for process in processes:
|
|
if process.returncode is None:
|
|
process.kill()
|
|
await process.wait()
|
|
after = await contents(empty_database)
|
|
assert {name: after[name] for name in before} == before
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
|
|
|
|
async def test_sender_must_be_member_even_bypassing_api() -> None:
|
|
with pytest.raises(IntegrityError) as failure:
|
|
async with sql_connection() as connection:
|
|
await connection.execute(
|
|
insert(MessageRow).values(
|
|
id=uuid4(), chat_id=uuid4(), sender_id=uuid4(), text="x", created_at=func.now()
|
|
)
|
|
)
|
|
assert getattr(failure.value.orig, "sqlstate", None) == "23503"
|
|
|
|
|
|
async def test_key_needs_a_member_and_a_message() -> None:
|
|
with pytest.raises(IntegrityError) as failure:
|
|
async with sql_connection() as connection:
|
|
await connection.execute(
|
|
insert(IdempotencyKeyRow).values(
|
|
chat_id=uuid4(),
|
|
user_id=uuid4(),
|
|
key="k-1",
|
|
fingerprint=b"x" * 32,
|
|
message_id=uuid4(),
|
|
created_at=func.now(),
|
|
)
|
|
)
|
|
assert getattr(failure.value.orig, "sqlstate", None) == "23503"
|
|
|
|
|
|
async def test_failed_revision_rolls_back_ddl_and_version(
|
|
empty_database: str, tmp_path: Path
|
|
) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
scripts = tmp_path / "alembic"
|
|
shutil.copytree(ROOT / "alembic", scripts, ignore=shutil.ignore_patterns("__pycache__"))
|
|
(scripts / "versions" / "0003_failure.py").write_text(
|
|
"from alembic import op\n"
|
|
"import sqlalchemy as sa\n"
|
|
"revision = '0003_failure'\n"
|
|
"down_revision = '0004_attachment_quarantine'\n"
|
|
"def upgrade():\n"
|
|
" op.create_table('must_rollback', sa.Column('id', sa.Integer, primary_key=True))\n"
|
|
" raise RuntimeError('migration failed')\n"
|
|
)
|
|
config = migration_config(empty_database)
|
|
config.set_main_option("script_location", str(scripts))
|
|
with pytest.raises(RuntimeError, match="migration failed"):
|
|
await asyncio.to_thread(command.upgrade, config, "head")
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
assert await contents(empty_database) == before
|
|
async with sql_connection(empty_database) as connection:
|
|
assert not await connection.run_sync(lambda sync: inspect(sync).has_table("must_rollback"))
|
|
|
|
|
|
async def test_attachment_owner_must_be_member() -> None:
|
|
with pytest.raises(IntegrityError, match="attachments_owner_fkey"):
|
|
async with sql_connection() as connection:
|
|
await connection.execute(
|
|
insert(AttachmentRow).values(
|
|
id=uuid4(),
|
|
chat_id=uuid4(),
|
|
owner_id=uuid4(),
|
|
filename="a.png",
|
|
content_type="image/png",
|
|
size_bytes=1,
|
|
sha256="0" * 64,
|
|
width=1,
|
|
height=1,
|
|
object_key="attachments/x",
|
|
status="pending",
|
|
created_at=func.now(),
|
|
updated_at=func.now(),
|
|
)
|
|
)
|
|
|
|
|
|
async def test_message_links_only_attachments_of_its_chat(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
with pytest.raises(IntegrityError, match="message_attachments_attachment_fkey"):
|
|
async with sql_connection(empty_database) as connection:
|
|
attachment = (await connection.execute(select(AttachmentRow.id))).scalar_one()
|
|
user = (await connection.execute(select(UserRow.id))).scalar_one()
|
|
chat, message = uuid4(), uuid4()
|
|
await connection.execute(
|
|
insert(ChatRow).values(id=chat, title="Other", created_at=func.now())
|
|
)
|
|
await connection.execute(
|
|
insert(ChatMemberRow).values(
|
|
chat_id=chat, user_id=user, position=1, chat_created_at=func.now()
|
|
)
|
|
)
|
|
await connection.execute(
|
|
insert(MessageRow).values(
|
|
id=message, chat_id=chat, sender_id=user, text="", created_at=func.now()
|
|
)
|
|
)
|
|
assert (
|
|
await connection.scalar(select(MessageRow.text).where(MessageRow.id == message))
|
|
== ""
|
|
)
|
|
await connection.execute(
|
|
insert(MessageAttachmentRow).values(
|
|
message_id=message,
|
|
position=0,
|
|
chat_id=chat,
|
|
attachment_id=attachment,
|
|
)
|
|
)
|