Files
mtusi2026lab2/student-tests/api/test_migrations.py
T

543 lines
21 KiB
Python

import asyncio
import hashlib
import os
import shutil
import sys
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from uuid import uuid4
import pytest
from alembic import command
from alembic.autogenerate import compare_metadata
from alembic.migration import MigrationContext
from alembic.script import ScriptDirectory
from alembic.util import CommandError
from sqlalchemy import MetaData, Table, case, func, insert, inspect, literal, make_url, select, text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import NullPool
from conftest import DATABASE, ROOT, migration_config, sql_connection, upgrade
from messenger.attachments.infrastructure.postgres import (
AttachmentRow,
JobEventRow,
JobRow,
OutboxRow,
VariantRow,
)
from messenger.identity.infrastructure.postgres import SessionRow, UserRow
from messenger.messaging.infrastructure.postgres import (
ChatMemberRow,
ChatRow,
IdempotencyKeyRow,
MessageAttachmentRow,
MessageRow,
)
from messenger.shared.infrastructure.postgres import Base
VERSIONS = [
"0001_init",
"0002_idempotency_keys",
"0003_attachments",
"0004_jobs",
"0005_failed_registry",
"0006_attachment_quarantine",
]
TABLES = [
UserRow,
SessionRow,
ChatRow,
ChatMemberRow,
MessageRow,
IdempotencyKeyRow,
AttachmentRow,
JobRow,
OutboxRow,
JobEventRow,
VariantRow,
MessageAttachmentRow,
]
@pytest.fixture
async def empty_database(database: str) -> AsyncIterator[str]:
name = f"migration_test_{uuid4().hex}"
url = make_url(database).set(drivername="postgresql+asyncpg")
engine = create_async_engine(url, isolation_level="AUTOCOMMIT", poolclass=NullPool)
try:
async with engine.connect() as connection:
await connection.exec_driver_sql(f"CREATE DATABASE {name}")
yield url.set(database=name).render_as_string(hide_password=False)
finally:
async with engine.connect() as connection:
await connection.exec_driver_sql(f"DROP DATABASE {name} WITH (FORCE)")
await engine.dispose()
async def legacy(dsn: str, versions: int = 6) -> None:
async with sql_connection(dsn) as connection:
await connection.execute(
text(
"CREATE TABLE schema_migrations (version text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT now())"
)
)
journal = await connection.run_sync(
lambda sync: Table("schema_migrations", MetaData(), autoload_with=sync)
)
for version in VERSIONS[:versions]:
sql = (ROOT / "student-tests" / "fixtures" / "legacy" / f"{version}.sql").read_text()
for statement in sql.split(";"):
if statement.strip():
await connection.execute(text(statement))
await connection.execute(
insert(journal).values(
version=version, checksum=hashlib.sha256(sql.encode()).hexdigest()
)
)
async def seed(dsn: str, versions: int = 6) -> None:
user, chat, message = uuid4(), uuid4(), uuid4()
stamp = datetime.now(UTC)
rows = [
(
UserRow,
{
"id": user,
"username": "legacy_user",
"display_name": "Старые данные",
"password_hash": "$argon2id$legacy",
"created_at": stamp,
},
),
(
SessionRow,
{
"token_hash": b"s" * 32,
"user_id": user,
"created_at": stamp,
"expires_at": stamp + timedelta(days=1),
},
),
(
ChatRow,
{
"id": chat,
"title": "Существующий чат",
"created_at": stamp,
"last_message_at": stamp,
},
),
(
ChatMemberRow,
{"chat_id": chat, "user_id": user, "position": 1, "chat_created_at": stamp},
),
(
MessageRow,
{
"id": message,
"chat_id": chat,
"sender_id": user,
"text": "Сохранить",
"created_at": stamp,
},
),
(
IdempotencyKeyRow,
{
"chat_id": chat,
"user_id": user,
"key": "legacy-key",
"fingerprint": b"f" * 32,
"message_id": message,
"created_at": stamp,
},
),
]
async with sql_connection(dsn) as connection:
for model, row in rows[: 5 if versions == 1 else 6]:
await connection.execute(insert(model).values(**row))
if versions >= 3:
attachment = uuid4()
row = {
"id": attachment,
"chat_id": chat,
"owner_id": user,
"filename": "legacy.png",
"content_type": "image/png",
"size_bytes": 17,
"sha256": "a" * 64,
"width": 1,
"height": 1,
"object_key": f"attachments/{attachment}",
"status": "ready",
"created_at": stamp,
"updated_at": stamp,
}
if versions >= 4:
row.update(original_key=row["object_key"], published_version=1)
if versions >= 6:
row.update(quarantine_reason="missing", quarantined_at=stamp)
table = await connection.run_sync(
lambda sync: Table("attachments", MetaData(), autoload_with=sync)
)
await connection.execute(insert(table).values(**row))
await connection.execute(
insert(MessageAttachmentRow).values(
message_id=message, position=0, chat_id=chat, attachment_id=attachment
)
)
if versions >= 4:
job = uuid4()
await connection.execute(
insert(JobRow).values(
job_id=job,
attachment_id=attachment,
pipeline_version=1,
status="processing",
attempt=2,
claims=3,
lease_token=uuid4(),
lease_until=stamp + timedelta(minutes=1),
created_at=stamp,
updated_at=stamp,
)
)
await connection.execute(
insert(OutboxRow).values(
job_id=job,
payload={"job_id": str(job)},
available_at=stamp,
)
)
await connection.execute(
insert(JobEventRow).values(
job_id=job,
attachment_id=attachment,
kind="claimed",
attempt=2,
)
)
await connection.execute(
insert(VariantRow).values(
attachment_id=attachment,
pipeline_version=1,
name="thumbnail",
object_key=f"variants/{attachment}/1/thumbnail.jpg",
content_type="image/jpeg",
size_bytes=17,
width=1,
height=1,
sha256="a" * 64,
created_at=stamp,
)
)
async def contents(dsn: str) -> dict:
async with sql_connection(dsn) as connection:
tables = await connection.run_sync(lambda sync: inspect(sync).get_table_names())
result = {}
for model in TABLES:
if model.__tablename__ not in tables:
continue
table = await connection.run_sync(
lambda sync, name=model.__tablename__: Table(name, MetaData(), autoload_with=sync)
)
missing = {}
if model is AttachmentRow and "published_version" not in table.c:
missing = {
"original_key": case((table.c.status == "ready", table.c.object_key)),
"staging_key": case((table.c.status == "pending", table.c.object_key)),
"published_version": case((table.c.status == "ready", 1), else_=0),
}
columns = [
table.c[col.name]
if col.name in table.c
else missing.get(col.name, literal(None)).label(col.name)
for col in model.__table__.columns
]
result[model.__tablename__] = (
await connection.execute(select(*columns).order_by(*table.primary_key))
).all()
return result
async def revision(dsn: str) -> str | None:
async with sql_connection(dsn) as connection:
return await connection.run_sync(
lambda sync: MigrationContext.configure(sync).get_current_revision()
)
async def test_migrations_are_ordered_and_versioned() -> None:
scripts = ScriptDirectory.from_config(migration_config(DATABASE["url"]))
assert [script.revision for script in reversed(list(scripts.walk_revisions()))] == VERSIONS
assert await revision(DATABASE["url"]) == VERSIONS[-1]
async def test_fresh_schema_matches_models(empty_database: str) -> None:
await upgrade(empty_database)
async with sql_connection(empty_database) as connection:
assert (
await connection.run_sync(
lambda sync: compare_metadata(MigrationContext.configure(sync), Base.metadata)
)
== []
)
await asyncio.to_thread(command.check, migration_config(empty_database))
async def test_rerun_changes_nothing(empty_database: str) -> None:
await upgrade(empty_database)
await seed(empty_database)
before = await contents(empty_database)
await upgrade(empty_database)
assert await contents(empty_database) == before
assert await revision(empty_database) == VERSIONS[-1]
@pytest.mark.parametrize("versions", [1, 2, 3, 4, 5, 6])
async def test_legacy_upgrade_preserves_every_row(empty_database: str, versions: int) -> None:
await legacy(empty_database, versions)
await seed(empty_database, versions)
before = await contents(empty_database)
async with sql_connection(empty_database) as connection:
journal = (
await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version"))
).all()
await upgrade(empty_database)
after = await contents(empty_database)
assert {name: after[name] for name in before} == before
assert await revision(empty_database) == VERSIONS[-1]
async with sql_connection(empty_database) as connection:
assert (
await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version"))
).all() == journal
assert (
await connection.scalar(
text("SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'alembic_verify_%'")
)
== 0
)
await upgrade(empty_database)
assert await contents(empty_database) == after
@pytest.mark.parametrize(
"damage",
[
"DROP INDEX messages_history_idx",
"ALTER TABLE messages DROP CONSTRAINT messages_chat_id_sender_id_fkey",
"ALTER TABLE users DROP CONSTRAINT users_username_format",
"ALTER TABLE users ALTER COLUMN display_name TYPE varchar",
'ALTER TABLE users ALTER COLUMN username TYPE text COLLATE "C"',
"ALTER TABLE sessions SET UNLOGGED",
"ALTER TABLE sessions ALTER COLUMN expires_at DROP NOT NULL",
"UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0001_init'",
"DELETE FROM schema_migrations WHERE version = '0001_init'",
"UPDATE schema_migrations SET version = 'unknown' WHERE version = '0002_idempotency_keys'",
"DROP TABLE schema_migrations",
"ALTER TABLE attachments DROP CONSTRAINT attachments_owner_fkey",
"ALTER TABLE attachments DROP CONSTRAINT attachments_quarantine_check",
"ALTER TABLE message_attachments DROP CONSTRAINT message_attachments_attachment_fkey",
'ALTER TABLE attachments ALTER COLUMN object_key TYPE text COLLATE "default"',
"DROP INDEX attachments_pending_idx",
"UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0003_attachments'",
],
)
async def test_legacy_drift_is_refused_without_stamping(empty_database: str, damage: str) -> None:
await legacy(empty_database)
await seed(empty_database)
before = await contents(empty_database)
async with sql_connection(empty_database) as connection:
await connection.execute(text(damage))
with pytest.raises(RuntimeError, match=r"[Ll]egacy|[Ee]xisting|Unknown"):
await upgrade(empty_database)
assert await revision(empty_database) is None
assert await contents(empty_database) == before
async def test_unknown_revision_is_refused(empty_database: str) -> None:
await upgrade(empty_database)
await seed(empty_database)
before = await contents(empty_database)
async with sql_connection(empty_database) as connection:
await connection.execute(text("UPDATE alembic_version SET version_num = 'unknown'"))
with pytest.raises(CommandError, match="unknown"):
await upgrade(empty_database)
assert await contents(empty_database) == before
assert await revision(empty_database) == "unknown"
@pytest.mark.parametrize("existing", [False, True])
async def test_simultaneous_migration_processes_serialize(
empty_database: str, existing: bool
) -> None:
if existing:
await legacy(empty_database)
await seed(empty_database)
before = await contents(empty_database)
processes = []
try:
for _ in range(2):
processes.append(
await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"alembic",
"upgrade",
"head",
cwd=ROOT,
env={**os.environ, "DATABASE_URL": empty_database},
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
)
async with asyncio.timeout(30):
output = await asyncio.gather(*(process.communicate() for process in processes))
assert [process.returncode for process in processes] == [0, 0], output
finally:
for process in processes:
if process.returncode is None:
process.kill()
await process.wait()
after = await contents(empty_database)
assert {name: after[name] for name in before} == before
assert await revision(empty_database) == VERSIONS[-1]
async def test_sender_must_be_member_even_bypassing_api() -> None:
with pytest.raises(IntegrityError) as failure:
async with sql_connection() as connection:
await connection.execute(
insert(MessageRow).values(
id=uuid4(), chat_id=uuid4(), sender_id=uuid4(), text="x", created_at=func.now()
)
)
assert getattr(failure.value.orig, "sqlstate", None) == "23503"
async def test_key_needs_a_member_and_a_message() -> None:
with pytest.raises(IntegrityError) as failure:
async with sql_connection() as connection:
await connection.execute(
insert(IdempotencyKeyRow).values(
chat_id=uuid4(),
user_id=uuid4(),
key="k-1",
fingerprint=b"x" * 32,
message_id=uuid4(),
created_at=func.now(),
)
)
assert getattr(failure.value.orig, "sqlstate", None) == "23503"
async def test_failed_revision_rolls_back_ddl_and_version(
empty_database: str, tmp_path: Path
) -> None:
await upgrade(empty_database)
await seed(empty_database)
before = await contents(empty_database)
scripts = tmp_path / "alembic"
shutil.copytree(ROOT / "alembic", scripts, ignore=shutil.ignore_patterns("__pycache__"))
(scripts / "versions" / "0003_failure.py").write_text(
"from alembic import op\n"
"import sqlalchemy as sa\n"
"revision = '0003_failure'\n"
"down_revision = '0006_attachment_quarantine'\n"
"def upgrade():\n"
" op.create_table('must_rollback', sa.Column('id', sa.Integer, primary_key=True))\n"
" raise RuntimeError('migration failed')\n"
)
config = migration_config(empty_database)
config.set_main_option("script_location", str(scripts))
with pytest.raises(RuntimeError, match="migration failed"):
await asyncio.to_thread(command.upgrade, config, "head")
assert await revision(empty_database) == VERSIONS[-1]
assert await contents(empty_database) == before
async with sql_connection(empty_database) as connection:
assert not await connection.run_sync(lambda sync: inspect(sync).has_table("must_rollback"))
async def test_attachment_owner_must_be_member() -> None:
with pytest.raises(IntegrityError, match="attachments_owner_fkey"):
async with sql_connection() as connection:
await connection.execute(
insert(AttachmentRow).values(
id=uuid4(),
chat_id=uuid4(),
owner_id=uuid4(),
filename="a.png",
content_type="image/png",
size_bytes=1,
sha256="0" * 64,
width=1,
height=1,
object_key="attachments/x",
status="pending",
created_at=func.now(),
updated_at=func.now(),
)
)
async def test_message_links_only_attachments_of_its_chat(empty_database: str) -> None:
await upgrade(empty_database)
await seed(empty_database)
with pytest.raises(IntegrityError, match="message_attachments_attachment_fkey"):
async with sql_connection(empty_database) as connection:
attachment = (await connection.execute(select(AttachmentRow.id))).scalar_one()
user = (await connection.execute(select(UserRow.id))).scalar_one()
chat, message = uuid4(), uuid4()
await connection.execute(
insert(ChatRow).values(id=chat, title="Other", created_at=func.now())
)
await connection.execute(
insert(ChatMemberRow).values(
chat_id=chat, user_id=user, position=1, chat_created_at=func.now()
)
)
await connection.execute(
insert(MessageRow).values(
id=message, chat_id=chat, sender_id=user, text="", created_at=func.now()
)
)
assert (
await connection.scalar(select(MessageRow.text).where(MessageRow.id == message))
== ""
)
await connection.execute(
insert(MessageAttachmentRow).values(
message_id=message,
position=0,
chat_id=chat,
attachment_id=attachment,
)
)
async def test_pipeline_tables_have_the_columns_the_suite_reads(empty_database: str) -> None:
await upgrade(empty_database)
async with sql_connection(empty_database) as connection:
for model in (JobRow, OutboxRow, VariantRow, JobEventRow):
columns = await connection.run_sync(
lambda sync, name=model.__tablename__: inspect(sync).get_columns(name)
)
assert {col["name"] for col in columns} == set(model.__table__.columns.keys())
async def test_one_variant_row_per_effect_key(empty_database: str) -> None:
await upgrade(empty_database)
await seed(empty_database)
async with sql_connection(empty_database) as connection:
row = (await connection.execute(select(VariantRow.__table__))).mappings().one()
duplicate = dict(row, object_key="variants/duplicate")
with pytest.raises(IntegrityError, match="attachment_variants_pkey"):
async with sql_connection(empty_database) as connection:
await connection.execute(insert(VariantRow).values(**duplicate))