diff --git a/openspec/changes/add-lab5-async-attachments/tasks.md b/openspec/changes/add-lab5-async-attachments/tasks.md index 4d6b81c9..9d2849ec 100644 --- a/openspec/changes/add-lab5-async-attachments/tasks.md +++ b/openspec/changes/add-lab5-async-attachments/tasks.md @@ -1,7 +1,7 @@ ## 1. Схема -- [ ] 1.1 Миграция `0004_jobs.sql`: статусы `queued/processing/failed`, `staging_key`, `original_key`, `published_version`, `error_code` у `attachments` (перенос `object_key` лабы 4), таблицы `attachment_variants`, `jobs`, `outbox`, `job_events`, частичные индексы `outbox_pending_idx`, `jobs_lease_idx`, `attachments_staged_idx` -- [ ] 1.2 Проверка группы: тест миграций — версия `0004`, повтор без изменений, колонки из `.prep/lab5/tools/schema.reference.sql` на месте, дубль ключа эффекта отклоняется первичным ключом +- [x] 1.1 Миграция `0004_jobs.sql`: статусы `queued/processing/failed`, `staging_key`, `original_key`, `published_version`, `error_code` у `attachments` (перенос `object_key` лабы 4), таблицы `attachment_variants`, `jobs`, `outbox`, `job_events`, частичные индексы `outbox_pending_idx`, `jobs_lease_idx`, `attachments_staged_idx` +- [x] 1.2 Проверка группы: тест миграций — версия `0004`, повтор без изменений, колонки из `.prep/lab5/tools/schema.reference.sql` на месте, дубль ключа эффекта отклоняется первичным ключом ## 2. Зона `attachments`: домен и приложение diff --git a/src/messenger/migrations/0004_jobs.sql b/src/messenger/migrations/0004_jobs.sql new file mode 100644 index 00000000..41e2b832 --- /dev/null +++ b/src/messenger/migrations/0004_jobs.sql @@ -0,0 +1,70 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status_check, + ADD CONSTRAINT attachments_status_check + CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed')), + ADD COLUMN staging_key text COLLATE "C" CONSTRAINT attachments_staging_key_key UNIQUE, + ADD COLUMN original_key text COLLATE "C" CONSTRAINT attachments_original_key_key UNIQUE, + ADD COLUMN published_version integer NOT NULL DEFAULT 0 CHECK (published_version >= 0), + ADD COLUMN error_code text + CHECK (error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')), + ALTER COLUMN object_key DROP NOT NULL; + +UPDATE attachments SET original_key = object_key, published_version = 1 WHERE status = 'ready'; +UPDATE attachments SET staging_key = object_key WHERE status = 'pending'; + +CREATE INDEX attachments_staged_idx ON attachments (updated_at) WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text COLLATE "C" NOT NULL CONSTRAINT attachment_variants_object_key_key UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'done', 'failed', 'superseded')), + attempt integer NOT NULL CHECK (attempt >= 1), + claims integer NOT NULL DEFAULT 0 CHECK (claims >= 0), + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz, + last_error text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT jobs_effect_key UNIQUE (attachment_id, pipeline_version) +); + +CREATE INDEX jobs_lease_idx ON jobs (lease_until) WHERE status = 'processing'; + +CREATE TABLE outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (job_id), + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + published_at timestamptz +); + +CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_job_idx ON outbox (job_id); + +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL, + kind text NOT NULL, + attempt integer, + detail jsonb NOT NULL DEFAULT '{}', + at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX job_events_attachment_idx ON job_events (attachment_id, id); diff --git a/student-tests/api/test_migrations.py b/student-tests/api/test_migrations.py index 9fa6e7ef..db11500c 100644 --- a/student-tests/api/test_migrations.py +++ b/student-tests/api/test_migrations.py @@ -10,8 +10,31 @@ COUNTS = """ SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM sessions), (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members), (SELECT count(*) FROM messages), (SELECT count(*) FROM idempotency_keys), - (SELECT count(*) FROM attachments), (SELECT count(*) FROM message_attachments) + (SELECT count(*) FROM attachments), (SELECT count(*) FROM message_attachments), + (SELECT count(*) FROM attachment_variants), (SELECT count(*) FROM jobs), + (SELECT count(*) FROM outbox), (SELECT count(*) FROM job_events) """ +PIPELINE_COLUMNS = { + "attachments": { + "id", + "status", + "published_version", + "error_code", + "staging_key", + "original_key", + }, + "attachment_variants": { + "attachment_id", + "pipeline_version", + "name", + "sha256", + "size_bytes", + "object_key", + }, + "jobs": {"job_id", "attachment_id", "pipeline_version", "status", "attempt"}, + "outbox": {"job_id", "payload", "published_at"}, + "job_events": {"job_id", "attachment_id", "kind", "attempt"}, +} async def counts() -> list[int]: @@ -25,7 +48,12 @@ async def counts() -> list[int]: async def test_migrations_are_ordered_and_versioned() -> None: versions = [version for version, _ in migrations()] assert versions == sorted(versions) - assert versions[:3] == ["0001_init", "0002_idempotency_keys", "0003_attachments"] + assert versions[:4] == [ + "0001_init", + "0002_idempotency_keys", + "0003_attachments", + "0004_jobs", + ] applied = await scalar("SELECT array_agg(version ORDER BY version) FROM schema_migrations") assert applied == versions @@ -149,3 +177,61 @@ async def test_message_links_only_attachments_of_its_chat() -> None: finally: await transaction.rollback() await connection.close() + + +async def test_pipeline_tables_have_the_columns_the_suite_reads() -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + rows = await connection.fetch( + "SELECT table_name, column_name FROM information_schema.columns " + "WHERE table_schema = current_schema()" + ) + finally: + await connection.close() + found: dict[str, set[str]] = {} + for row in rows: + found.setdefault(row["table_name"], set()).add(row["column_name"]) + assert all(columns <= found.get(table, set()) for table, columns in PIPELINE_COLUMNS.items()) + + +async def test_one_variant_row_per_effect_key() -> None: + user, chat, attachment = uuid4(), uuid4(), uuid4() + variant = ( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, " + "content_type, size_bytes, width, height, sha256, created_at) " + "VALUES ($1, 1, 'thumbnail', $2, 'image/jpeg', 1, 1, 1, repeat('a', 64), now())" + ) + connection = await asyncpg.connect(DATABASE["url"]) + transaction = connection.transaction() + await transaction.start() + try: + await connection.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at) " + "VALUES ($1, $2, 'A', '$argon2id$x', now())", + user, + f"u_{user.hex[:12]}", + ) + await connection.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, 'A', now())", chat + ) + await connection.execute( + "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " + "VALUES ($1, $2, 1, now())", + chat, + user, + ) + await connection.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, " + "sha256, width, height, staging_key, status, created_at, updated_at) VALUES ($1, $2, " + "$3, 'a.png', 'image/png', 1, repeat('0', 64), 1, 1, $4, 'queued', now(), now())", + attachment, + chat, + user, + f"staging/{attachment}", + ) + await connection.execute(variant, attachment, f"variants/{attachment}/1/a") + with pytest.raises(asyncpg.UniqueViolationError, match="attachment_variants_pkey"): + await connection.execute(variant, attachment, f"variants/{attachment}/1/b") + finally: + await transaction.rollback() + await connection.close()