forked from mtusi2026/mtusi2026lab2
543 lines
21 KiB
Python
543 lines
21 KiB
Python
import asyncio
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import sys
|
|
from collections.abc import AsyncIterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from alembic import command
|
|
from alembic.autogenerate import compare_metadata
|
|
from alembic.migration import MigrationContext
|
|
from alembic.script import ScriptDirectory
|
|
from alembic.util import CommandError
|
|
from sqlalchemy import MetaData, Table, case, func, insert, inspect, literal, make_url, select, text
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from sqlalchemy.pool import NullPool
|
|
|
|
from conftest import DATABASE, ROOT, migration_config, sql_connection, upgrade
|
|
from messenger.attachments.infrastructure.postgres import (
|
|
AttachmentRow,
|
|
JobEventRow,
|
|
JobRow,
|
|
OutboxRow,
|
|
VariantRow,
|
|
)
|
|
from messenger.identity.infrastructure.postgres import SessionRow, UserRow
|
|
from messenger.messaging.infrastructure.postgres import (
|
|
ChatMemberRow,
|
|
ChatRow,
|
|
IdempotencyKeyRow,
|
|
MessageAttachmentRow,
|
|
MessageRow,
|
|
)
|
|
from messenger.shared.infrastructure.postgres import Base
|
|
|
|
VERSIONS = [
|
|
"0001_init",
|
|
"0002_idempotency_keys",
|
|
"0003_attachments",
|
|
"0004_jobs",
|
|
"0005_failed_registry",
|
|
"0006_attachment_quarantine",
|
|
]
|
|
TABLES = [
|
|
UserRow,
|
|
SessionRow,
|
|
ChatRow,
|
|
ChatMemberRow,
|
|
MessageRow,
|
|
IdempotencyKeyRow,
|
|
AttachmentRow,
|
|
JobRow,
|
|
OutboxRow,
|
|
JobEventRow,
|
|
VariantRow,
|
|
MessageAttachmentRow,
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
async def empty_database(database: str) -> AsyncIterator[str]:
|
|
name = f"migration_test_{uuid4().hex}"
|
|
url = make_url(database).set(drivername="postgresql+asyncpg")
|
|
engine = create_async_engine(url, isolation_level="AUTOCOMMIT", poolclass=NullPool)
|
|
try:
|
|
async with engine.connect() as connection:
|
|
await connection.exec_driver_sql(f"CREATE DATABASE {name}")
|
|
yield url.set(database=name).render_as_string(hide_password=False)
|
|
finally:
|
|
async with engine.connect() as connection:
|
|
await connection.exec_driver_sql(f"DROP DATABASE {name} WITH (FORCE)")
|
|
await engine.dispose()
|
|
|
|
|
|
async def legacy(dsn: str, versions: int = 6) -> None:
|
|
async with sql_connection(dsn) as connection:
|
|
await connection.execute(
|
|
text(
|
|
"CREATE TABLE schema_migrations (version text PRIMARY KEY, checksum text NOT NULL, "
|
|
"applied_at timestamptz NOT NULL DEFAULT now())"
|
|
)
|
|
)
|
|
journal = await connection.run_sync(
|
|
lambda sync: Table("schema_migrations", MetaData(), autoload_with=sync)
|
|
)
|
|
for version in VERSIONS[:versions]:
|
|
sql = (ROOT / "student-tests" / "fixtures" / "legacy" / f"{version}.sql").read_text()
|
|
for statement in sql.split(";"):
|
|
if statement.strip():
|
|
await connection.execute(text(statement))
|
|
await connection.execute(
|
|
insert(journal).values(
|
|
version=version, checksum=hashlib.sha256(sql.encode()).hexdigest()
|
|
)
|
|
)
|
|
|
|
|
|
async def seed(dsn: str, versions: int = 6) -> None:
|
|
user, chat, message = uuid4(), uuid4(), uuid4()
|
|
stamp = datetime.now(UTC)
|
|
rows = [
|
|
(
|
|
UserRow,
|
|
{
|
|
"id": user,
|
|
"username": "legacy_user",
|
|
"display_name": "Старые данные",
|
|
"password_hash": "$argon2id$legacy",
|
|
"created_at": stamp,
|
|
},
|
|
),
|
|
(
|
|
SessionRow,
|
|
{
|
|
"token_hash": b"s" * 32,
|
|
"user_id": user,
|
|
"created_at": stamp,
|
|
"expires_at": stamp + timedelta(days=1),
|
|
},
|
|
),
|
|
(
|
|
ChatRow,
|
|
{
|
|
"id": chat,
|
|
"title": "Существующий чат",
|
|
"created_at": stamp,
|
|
"last_message_at": stamp,
|
|
},
|
|
),
|
|
(
|
|
ChatMemberRow,
|
|
{"chat_id": chat, "user_id": user, "position": 1, "chat_created_at": stamp},
|
|
),
|
|
(
|
|
MessageRow,
|
|
{
|
|
"id": message,
|
|
"chat_id": chat,
|
|
"sender_id": user,
|
|
"text": "Сохранить",
|
|
"created_at": stamp,
|
|
},
|
|
),
|
|
(
|
|
IdempotencyKeyRow,
|
|
{
|
|
"chat_id": chat,
|
|
"user_id": user,
|
|
"key": "legacy-key",
|
|
"fingerprint": b"f" * 32,
|
|
"message_id": message,
|
|
"created_at": stamp,
|
|
},
|
|
),
|
|
]
|
|
async with sql_connection(dsn) as connection:
|
|
for model, row in rows[: 5 if versions == 1 else 6]:
|
|
await connection.execute(insert(model).values(**row))
|
|
if versions >= 3:
|
|
attachment = uuid4()
|
|
row = {
|
|
"id": attachment,
|
|
"chat_id": chat,
|
|
"owner_id": user,
|
|
"filename": "legacy.png",
|
|
"content_type": "image/png",
|
|
"size_bytes": 17,
|
|
"sha256": "a" * 64,
|
|
"width": 1,
|
|
"height": 1,
|
|
"object_key": f"attachments/{attachment}",
|
|
"status": "ready",
|
|
"created_at": stamp,
|
|
"updated_at": stamp,
|
|
}
|
|
if versions >= 4:
|
|
row.update(original_key=row["object_key"], published_version=1)
|
|
if versions >= 6:
|
|
row.update(quarantine_reason="missing", quarantined_at=stamp)
|
|
table = await connection.run_sync(
|
|
lambda sync: Table("attachments", MetaData(), autoload_with=sync)
|
|
)
|
|
await connection.execute(insert(table).values(**row))
|
|
await connection.execute(
|
|
insert(MessageAttachmentRow).values(
|
|
message_id=message, position=0, chat_id=chat, attachment_id=attachment
|
|
)
|
|
)
|
|
if versions >= 4:
|
|
job = uuid4()
|
|
await connection.execute(
|
|
insert(JobRow).values(
|
|
job_id=job,
|
|
attachment_id=attachment,
|
|
pipeline_version=1,
|
|
status="processing",
|
|
attempt=2,
|
|
claims=3,
|
|
lease_token=uuid4(),
|
|
lease_until=stamp + timedelta(minutes=1),
|
|
created_at=stamp,
|
|
updated_at=stamp,
|
|
)
|
|
)
|
|
await connection.execute(
|
|
insert(OutboxRow).values(
|
|
job_id=job,
|
|
payload={"job_id": str(job)},
|
|
available_at=stamp,
|
|
)
|
|
)
|
|
await connection.execute(
|
|
insert(JobEventRow).values(
|
|
job_id=job,
|
|
attachment_id=attachment,
|
|
kind="claimed",
|
|
attempt=2,
|
|
)
|
|
)
|
|
await connection.execute(
|
|
insert(VariantRow).values(
|
|
attachment_id=attachment,
|
|
pipeline_version=1,
|
|
name="thumbnail",
|
|
object_key=f"variants/{attachment}/1/thumbnail.jpg",
|
|
content_type="image/jpeg",
|
|
size_bytes=17,
|
|
width=1,
|
|
height=1,
|
|
sha256="a" * 64,
|
|
created_at=stamp,
|
|
)
|
|
)
|
|
|
|
|
|
async def contents(dsn: str) -> dict:
|
|
async with sql_connection(dsn) as connection:
|
|
tables = await connection.run_sync(lambda sync: inspect(sync).get_table_names())
|
|
result = {}
|
|
for model in TABLES:
|
|
if model.__tablename__ not in tables:
|
|
continue
|
|
table = await connection.run_sync(
|
|
lambda sync, name=model.__tablename__: Table(name, MetaData(), autoload_with=sync)
|
|
)
|
|
missing = {}
|
|
if model is AttachmentRow and "published_version" not in table.c:
|
|
missing = {
|
|
"original_key": case((table.c.status == "ready", table.c.object_key)),
|
|
"staging_key": case((table.c.status == "pending", table.c.object_key)),
|
|
"published_version": case((table.c.status == "ready", 1), else_=0),
|
|
}
|
|
columns = [
|
|
table.c[col.name]
|
|
if col.name in table.c
|
|
else missing.get(col.name, literal(None)).label(col.name)
|
|
for col in model.__table__.columns
|
|
]
|
|
result[model.__tablename__] = (
|
|
await connection.execute(select(*columns).order_by(*table.primary_key))
|
|
).all()
|
|
return result
|
|
|
|
|
|
async def revision(dsn: str) -> str | None:
|
|
async with sql_connection(dsn) as connection:
|
|
return await connection.run_sync(
|
|
lambda sync: MigrationContext.configure(sync).get_current_revision()
|
|
)
|
|
|
|
|
|
async def test_migrations_are_ordered_and_versioned() -> None:
|
|
scripts = ScriptDirectory.from_config(migration_config(DATABASE["url"]))
|
|
assert [script.revision for script in reversed(list(scripts.walk_revisions()))] == VERSIONS
|
|
assert await revision(DATABASE["url"]) == VERSIONS[-1]
|
|
|
|
|
|
async def test_fresh_schema_matches_models(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
assert (
|
|
await connection.run_sync(
|
|
lambda sync: compare_metadata(MigrationContext.configure(sync), Base.metadata)
|
|
)
|
|
== []
|
|
)
|
|
await asyncio.to_thread(command.check, migration_config(empty_database))
|
|
|
|
|
|
async def test_rerun_changes_nothing(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
await upgrade(empty_database)
|
|
assert await contents(empty_database) == before
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
|
|
|
|
@pytest.mark.parametrize("versions", [1, 2, 3, 4, 5, 6])
|
|
async def test_legacy_upgrade_preserves_every_row(empty_database: str, versions: int) -> None:
|
|
await legacy(empty_database, versions)
|
|
await seed(empty_database, versions)
|
|
before = await contents(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
journal = (
|
|
await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version"))
|
|
).all()
|
|
await upgrade(empty_database)
|
|
after = await contents(empty_database)
|
|
assert {name: after[name] for name in before} == before
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
async with sql_connection(empty_database) as connection:
|
|
assert (
|
|
await connection.execute(text("SELECT * FROM schema_migrations ORDER BY version"))
|
|
).all() == journal
|
|
assert (
|
|
await connection.scalar(
|
|
text("SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'alembic_verify_%'")
|
|
)
|
|
== 0
|
|
)
|
|
await upgrade(empty_database)
|
|
assert await contents(empty_database) == after
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"damage",
|
|
[
|
|
"DROP INDEX messages_history_idx",
|
|
"ALTER TABLE messages DROP CONSTRAINT messages_chat_id_sender_id_fkey",
|
|
"ALTER TABLE users DROP CONSTRAINT users_username_format",
|
|
"ALTER TABLE users ALTER COLUMN display_name TYPE varchar",
|
|
'ALTER TABLE users ALTER COLUMN username TYPE text COLLATE "C"',
|
|
"ALTER TABLE sessions SET UNLOGGED",
|
|
"ALTER TABLE sessions ALTER COLUMN expires_at DROP NOT NULL",
|
|
"UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0001_init'",
|
|
"DELETE FROM schema_migrations WHERE version = '0001_init'",
|
|
"UPDATE schema_migrations SET version = 'unknown' WHERE version = '0002_idempotency_keys'",
|
|
"DROP TABLE schema_migrations",
|
|
"ALTER TABLE attachments DROP CONSTRAINT attachments_owner_fkey",
|
|
"ALTER TABLE attachments DROP CONSTRAINT attachments_quarantine_check",
|
|
"ALTER TABLE message_attachments DROP CONSTRAINT message_attachments_attachment_fkey",
|
|
'ALTER TABLE attachments ALTER COLUMN object_key TYPE text COLLATE "default"',
|
|
"DROP INDEX attachments_pending_idx",
|
|
"UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0003_attachments'",
|
|
],
|
|
)
|
|
async def test_legacy_drift_is_refused_without_stamping(empty_database: str, damage: str) -> None:
|
|
await legacy(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
await connection.execute(text(damage))
|
|
with pytest.raises(RuntimeError, match=r"[Ll]egacy|[Ee]xisting|Unknown"):
|
|
await upgrade(empty_database)
|
|
assert await revision(empty_database) is None
|
|
assert await contents(empty_database) == before
|
|
|
|
|
|
async def test_unknown_revision_is_refused(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
await connection.execute(text("UPDATE alembic_version SET version_num = 'unknown'"))
|
|
with pytest.raises(CommandError, match="unknown"):
|
|
await upgrade(empty_database)
|
|
assert await contents(empty_database) == before
|
|
assert await revision(empty_database) == "unknown"
|
|
|
|
|
|
@pytest.mark.parametrize("existing", [False, True])
|
|
async def test_simultaneous_migration_processes_serialize(
|
|
empty_database: str, existing: bool
|
|
) -> None:
|
|
if existing:
|
|
await legacy(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
processes = []
|
|
try:
|
|
for _ in range(2):
|
|
processes.append(
|
|
await asyncio.create_subprocess_exec(
|
|
sys.executable,
|
|
"-m",
|
|
"alembic",
|
|
"upgrade",
|
|
"head",
|
|
cwd=ROOT,
|
|
env={**os.environ, "DATABASE_URL": empty_database},
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
)
|
|
async with asyncio.timeout(30):
|
|
output = await asyncio.gather(*(process.communicate() for process in processes))
|
|
assert [process.returncode for process in processes] == [0, 0], output
|
|
finally:
|
|
for process in processes:
|
|
if process.returncode is None:
|
|
process.kill()
|
|
await process.wait()
|
|
after = await contents(empty_database)
|
|
assert {name: after[name] for name in before} == before
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
|
|
|
|
async def test_sender_must_be_member_even_bypassing_api() -> None:
|
|
with pytest.raises(IntegrityError) as failure:
|
|
async with sql_connection() as connection:
|
|
await connection.execute(
|
|
insert(MessageRow).values(
|
|
id=uuid4(), chat_id=uuid4(), sender_id=uuid4(), text="x", created_at=func.now()
|
|
)
|
|
)
|
|
assert getattr(failure.value.orig, "sqlstate", None) == "23503"
|
|
|
|
|
|
async def test_key_needs_a_member_and_a_message() -> None:
|
|
with pytest.raises(IntegrityError) as failure:
|
|
async with sql_connection() as connection:
|
|
await connection.execute(
|
|
insert(IdempotencyKeyRow).values(
|
|
chat_id=uuid4(),
|
|
user_id=uuid4(),
|
|
key="k-1",
|
|
fingerprint=b"x" * 32,
|
|
message_id=uuid4(),
|
|
created_at=func.now(),
|
|
)
|
|
)
|
|
assert getattr(failure.value.orig, "sqlstate", None) == "23503"
|
|
|
|
|
|
async def test_failed_revision_rolls_back_ddl_and_version(
|
|
empty_database: str, tmp_path: Path
|
|
) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
before = await contents(empty_database)
|
|
scripts = tmp_path / "alembic"
|
|
shutil.copytree(ROOT / "alembic", scripts, ignore=shutil.ignore_patterns("__pycache__"))
|
|
(scripts / "versions" / "0003_failure.py").write_text(
|
|
"from alembic import op\n"
|
|
"import sqlalchemy as sa\n"
|
|
"revision = '0003_failure'\n"
|
|
"down_revision = '0006_attachment_quarantine'\n"
|
|
"def upgrade():\n"
|
|
" op.create_table('must_rollback', sa.Column('id', sa.Integer, primary_key=True))\n"
|
|
" raise RuntimeError('migration failed')\n"
|
|
)
|
|
config = migration_config(empty_database)
|
|
config.set_main_option("script_location", str(scripts))
|
|
with pytest.raises(RuntimeError, match="migration failed"):
|
|
await asyncio.to_thread(command.upgrade, config, "head")
|
|
assert await revision(empty_database) == VERSIONS[-1]
|
|
assert await contents(empty_database) == before
|
|
async with sql_connection(empty_database) as connection:
|
|
assert not await connection.run_sync(lambda sync: inspect(sync).has_table("must_rollback"))
|
|
|
|
|
|
async def test_attachment_owner_must_be_member() -> None:
|
|
with pytest.raises(IntegrityError, match="attachments_owner_fkey"):
|
|
async with sql_connection() as connection:
|
|
await connection.execute(
|
|
insert(AttachmentRow).values(
|
|
id=uuid4(),
|
|
chat_id=uuid4(),
|
|
owner_id=uuid4(),
|
|
filename="a.png",
|
|
content_type="image/png",
|
|
size_bytes=1,
|
|
sha256="0" * 64,
|
|
width=1,
|
|
height=1,
|
|
object_key="attachments/x",
|
|
status="pending",
|
|
created_at=func.now(),
|
|
updated_at=func.now(),
|
|
)
|
|
)
|
|
|
|
|
|
async def test_message_links_only_attachments_of_its_chat(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
with pytest.raises(IntegrityError, match="message_attachments_attachment_fkey"):
|
|
async with sql_connection(empty_database) as connection:
|
|
attachment = (await connection.execute(select(AttachmentRow.id))).scalar_one()
|
|
user = (await connection.execute(select(UserRow.id))).scalar_one()
|
|
chat, message = uuid4(), uuid4()
|
|
await connection.execute(
|
|
insert(ChatRow).values(id=chat, title="Other", created_at=func.now())
|
|
)
|
|
await connection.execute(
|
|
insert(ChatMemberRow).values(
|
|
chat_id=chat, user_id=user, position=1, chat_created_at=func.now()
|
|
)
|
|
)
|
|
await connection.execute(
|
|
insert(MessageRow).values(
|
|
id=message, chat_id=chat, sender_id=user, text="", created_at=func.now()
|
|
)
|
|
)
|
|
assert (
|
|
await connection.scalar(select(MessageRow.text).where(MessageRow.id == message))
|
|
== ""
|
|
)
|
|
await connection.execute(
|
|
insert(MessageAttachmentRow).values(
|
|
message_id=message,
|
|
position=0,
|
|
chat_id=chat,
|
|
attachment_id=attachment,
|
|
)
|
|
)
|
|
|
|
|
|
async def test_pipeline_tables_have_the_columns_the_suite_reads(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
for model in (JobRow, OutboxRow, VariantRow, JobEventRow):
|
|
columns = await connection.run_sync(
|
|
lambda sync, name=model.__tablename__: inspect(sync).get_columns(name)
|
|
)
|
|
assert {col["name"] for col in columns} == set(model.__table__.columns.keys())
|
|
|
|
|
|
async def test_one_variant_row_per_effect_key(empty_database: str) -> None:
|
|
await upgrade(empty_database)
|
|
await seed(empty_database)
|
|
async with sql_connection(empty_database) as connection:
|
|
row = (await connection.execute(select(VariantRow.__table__))).mappings().one()
|
|
duplicate = dict(row, object_key="variants/duplicate")
|
|
with pytest.raises(IntegrityError, match="attachment_variants_pkey"):
|
|
async with sql_connection(empty_database) as connection:
|
|
await connection.execute(insert(VariantRow).values(**duplicate))
|