From 64fdaa03f246d3532aaf4074cdad6e840d695ce1 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 29 Oct 2026 00:43:34 +0300 Subject: [PATCH] Add the jobs, outbox and variants schema as an additive migration Migration 0004 widens the attachment states to queued, processing and failed, adds the staging and original keys, the published pipeline version and the error code, and moves the lab 4 object key into them while leaving the old column nullable until a contract step. New tables hold the variants under the effect key (attachment, pipeline version, name), the jobs with attempt, claims and lease, the outbox rows the relay publishes and the audit of job transitions. Partial indexes serve the relay scan of unpublished rows, the search for expired leases and the staging cleanup. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../add-lab5-async-attachments/tasks.md | 4 +- src/messenger/migrations/0004_jobs.sql | 70 +++++++++++++++ student-tests/api/test_migrations.py | 90 ++++++++++++++++++- 3 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 src/messenger/migrations/0004_jobs.sql diff --git a/openspec/changes/add-lab5-async-attachments/tasks.md b/openspec/changes/add-lab5-async-attachments/tasks.md index 4d6b81c9..9d2849ec 100644 --- a/openspec/changes/add-lab5-async-attachments/tasks.md +++ b/openspec/changes/add-lab5-async-attachments/tasks.md @@ -1,7 +1,7 @@ ## 1. Схема -- [ ] 1.1 Миграция `0004_jobs.sql`: статусы `queued/processing/failed`, `staging_key`, `original_key`, `published_version`, `error_code` у `attachments` (перенос `object_key` лабы 4), таблицы `attachment_variants`, `jobs`, `outbox`, `job_events`, частичные индексы `outbox_pending_idx`, `jobs_lease_idx`, `attachments_staged_idx` -- [ ] 1.2 Проверка группы: тест миграций — версия `0004`, повтор без изменений, колонки из `.prep/lab5/tools/schema.reference.sql` на месте, дубль ключа эффекта отклоняется первичным ключом +- [x] 1.1 Миграция `0004_jobs.sql`: статусы `queued/processing/failed`, `staging_key`, `original_key`, `published_version`, `error_code` у `attachments` (перенос `object_key` лабы 4), таблицы `attachment_variants`, `jobs`, `outbox`, `job_events`, частичные индексы `outbox_pending_idx`, `jobs_lease_idx`, `attachments_staged_idx` +- [x] 1.2 Проверка группы: тест миграций — версия `0004`, повтор без изменений, колонки из `.prep/lab5/tools/schema.reference.sql` на месте, дубль ключа эффекта отклоняется первичным ключом ## 2. Зона `attachments`: домен и приложение diff --git a/src/messenger/migrations/0004_jobs.sql b/src/messenger/migrations/0004_jobs.sql new file mode 100644 index 00000000..41e2b832 --- /dev/null +++ b/src/messenger/migrations/0004_jobs.sql @@ -0,0 +1,70 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status_check, + ADD CONSTRAINT attachments_status_check + CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed')), + ADD COLUMN staging_key text COLLATE "C" CONSTRAINT attachments_staging_key_key UNIQUE, + ADD COLUMN original_key text COLLATE "C" CONSTRAINT attachments_original_key_key UNIQUE, + ADD COLUMN published_version integer NOT NULL DEFAULT 0 CHECK (published_version >= 0), + ADD COLUMN error_code text + CHECK (error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')), + ALTER COLUMN object_key DROP NOT NULL; + +UPDATE attachments SET original_key = object_key, published_version = 1 WHERE status = 'ready'; +UPDATE attachments SET staging_key = object_key WHERE status = 'pending'; + +CREATE INDEX attachments_staged_idx ON attachments (updated_at) WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text COLLATE "C" NOT NULL CONSTRAINT attachment_variants_object_key_key UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'done', 'failed', 'superseded')), + attempt integer NOT NULL CHECK (attempt >= 1), + claims integer NOT NULL DEFAULT 0 CHECK (claims >= 0), + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz, + last_error text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT jobs_effect_key UNIQUE (attachment_id, pipeline_version) +); + +CREATE INDEX jobs_lease_idx ON jobs (lease_until) WHERE status = 'processing'; + +CREATE TABLE outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (job_id), + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + published_at timestamptz +); + +CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_job_idx ON outbox (job_id); + +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL, + kind text NOT NULL, + attempt integer, + detail jsonb NOT NULL DEFAULT '{}', + at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX job_events_attachment_idx ON job_events (attachment_id, id); diff --git a/student-tests/api/test_migrations.py b/student-tests/api/test_migrations.py index 9fa6e7ef..db11500c 100644 --- a/student-tests/api/test_migrations.py +++ b/student-tests/api/test_migrations.py @@ -10,8 +10,31 @@ COUNTS = """ SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM sessions), (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members), (SELECT count(*) FROM messages), (SELECT count(*) FROM idempotency_keys), - (SELECT count(*) FROM attachments), (SELECT count(*) FROM message_attachments) + (SELECT count(*) FROM attachments), (SELECT count(*) FROM message_attachments), + (SELECT count(*) FROM attachment_variants), (SELECT count(*) FROM jobs), + (SELECT count(*) FROM outbox), (SELECT count(*) FROM job_events) """ +PIPELINE_COLUMNS = { + "attachments": { + "id", + "status", + "published_version", + "error_code", + "staging_key", + "original_key", + }, + "attachment_variants": { + "attachment_id", + "pipeline_version", + "name", + "sha256", + "size_bytes", + "object_key", + }, + "jobs": {"job_id", "attachment_id", "pipeline_version", "status", "attempt"}, + "outbox": {"job_id", "payload", "published_at"}, + "job_events": {"job_id", "attachment_id", "kind", "attempt"}, +} async def counts() -> list[int]: @@ -25,7 +48,12 @@ async def counts() -> list[int]: async def test_migrations_are_ordered_and_versioned() -> None: versions = [version for version, _ in migrations()] assert versions == sorted(versions) - assert versions[:3] == ["0001_init", "0002_idempotency_keys", "0003_attachments"] + assert versions[:4] == [ + "0001_init", + "0002_idempotency_keys", + "0003_attachments", + "0004_jobs", + ] applied = await scalar("SELECT array_agg(version ORDER BY version) FROM schema_migrations") assert applied == versions @@ -149,3 +177,61 @@ async def test_message_links_only_attachments_of_its_chat() -> None: finally: await transaction.rollback() await connection.close() + + +async def test_pipeline_tables_have_the_columns_the_suite_reads() -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + rows = await connection.fetch( + "SELECT table_name, column_name FROM information_schema.columns " + "WHERE table_schema = current_schema()" + ) + finally: + await connection.close() + found: dict[str, set[str]] = {} + for row in rows: + found.setdefault(row["table_name"], set()).add(row["column_name"]) + assert all(columns <= found.get(table, set()) for table, columns in PIPELINE_COLUMNS.items()) + + +async def test_one_variant_row_per_effect_key() -> None: + user, chat, attachment = uuid4(), uuid4(), uuid4() + variant = ( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, " + "content_type, size_bytes, width, height, sha256, created_at) " + "VALUES ($1, 1, 'thumbnail', $2, 'image/jpeg', 1, 1, 1, repeat('a', 64), now())" + ) + connection = await asyncpg.connect(DATABASE["url"]) + transaction = connection.transaction() + await transaction.start() + try: + await connection.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at) " + "VALUES ($1, $2, 'A', '$argon2id$x', now())", + user, + f"u_{user.hex[:12]}", + ) + await connection.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, 'A', now())", chat + ) + await connection.execute( + "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " + "VALUES ($1, $2, 1, now())", + chat, + user, + ) + await connection.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, " + "sha256, width, height, staging_key, status, created_at, updated_at) VALUES ($1, $2, " + "$3, 'a.png', 'image/png', 1, repeat('0', 64), 1, 1, $4, 'queued', now(), now())", + attachment, + chat, + user, + f"staging/{attachment}", + ) + await connection.execute(variant, attachment, f"variants/{attachment}/1/a") + with pytest.raises(asyncpg.UniqueViolationError, match="attachment_variants_pkey"): + await connection.execute(variant, attachment, f"variants/{attachment}/1/b") + finally: + await transaction.rollback() + await connection.close()