forked from mtusi2026/mtusi2026lab7
134 lines
8.2 KiB
SQL
134 lines
8.2 KiB
SQL
-- lab5: асинхронная обработка вложений. Задания, outbox, варианты, журнал событий.
|
||
--
|
||
-- Состояния attachments.state:
|
||
-- uploading — строка-намерение до записи в staging (как в lab4), клиенту не видна;
|
||
-- queued / processing / ready / failed — публичные состояния из контракта.
|
||
-- source_key — исходник в staging (staging/<hex>; у старых вложений lab4 — originals/<hex>);
|
||
-- object_key — опубликованный оригинал attachments/<id>/v<N>/original, есть только у ready.
|
||
|
||
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
|
||
ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_check;
|
||
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
|
||
ALTER TABLE attachments ADD COLUMN source_key text UNIQUE;
|
||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||
ALTER TABLE attachments ADD COLUMN pipeline_version integer;
|
||
|
||
-- Вложения lab4 были приняты синхронно и не имеют вариантов: переносим их в staging-модель и ставим
|
||
-- в очередь (ниже), worker опубликует оригинал в конечный prefix и сделает варианты.
|
||
UPDATE attachments SET source_key = object_key, object_key = NULL;
|
||
UPDATE attachments SET state = 'queued', updated_at = now() WHERE state = 'ready';
|
||
|
||
ALTER TABLE attachments ADD CONSTRAINT attachments_state_check
|
||
CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed'));
|
||
ALTER TABLE attachments ADD CONSTRAINT attachments_source_key_check
|
||
CHECK (source_key ~ '^(staging|originals)/[a-f0-9]{32}$');
|
||
ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_check
|
||
CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/original$');
|
||
ALTER TABLE attachments ADD CONSTRAINT attachments_error_code_check
|
||
CHECK (error_code ~ '^[a-z_]{1,64}$');
|
||
-- ready <=> опубликованный оригинал; failed <=> код ошибки; до ready исходник обязан быть в staging.
|
||
ALTER TABLE attachments ADD CONSTRAINT attachments_ready_has_object
|
||
CHECK ((state = 'ready') = (object_key IS NOT NULL));
|
||
ALTER TABLE attachments ADD CONSTRAINT attachments_failed_has_code
|
||
CHECK ((state = 'failed') = (error_code IS NOT NULL));
|
||
ALTER TABLE attachments ADD CONSTRAINT attachments_pending_has_source
|
||
CHECK (state NOT IN ('uploading', 'queued', 'processing') OR source_key IS NOT NULL);
|
||
|
||
-- janitor: исходники ready/failed, которые пора удалить из staging
|
||
CREATE INDEX attachments_staging ON attachments (updated_at)
|
||
WHERE source_key IS NOT NULL AND state IN ('ready', 'failed');
|
||
|
||
-- Варианты. Первичный ключ = ключ эффекта (attachment_id, pipeline_version, variant):
|
||
-- повтор job не может создать второй набор вариантов.
|
||
CREATE TABLE attachment_variants (
|
||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||
name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||
content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
|
||
size_bytes integer NOT NULL CHECK (size_bytes >= 1),
|
||
width integer NOT NULL CHECK (width >= 1),
|
||
height integer NOT NULL CHECK (height >= 1),
|
||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||
object_key text NOT NULL UNIQUE
|
||
CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/[a-z]+\.(jpg|png|webp)$'),
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
PRIMARY KEY (attachment_id, pipeline_version, name)
|
||
);
|
||
|
||
-- Задание: одна логическая работа на (вложение, версия pipeline). job_id не меняется при повторах.
|
||
-- attempt — управляемая попытка (claim worker'ом); lease_* — аренда processing с fencing-токеном.
|
||
CREATE TABLE attachment_jobs (
|
||
job_id uuid PRIMARY KEY,
|
||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||
status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||
max_attempts integer NOT NULL CHECK (max_attempts >= 1),
|
||
available_at timestamptz NOT NULL DEFAULT now(),
|
||
lease_token uuid,
|
||
lease_owner text,
|
||
lease_until timestamptz,
|
||
error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'),
|
||
last_error text CHECK (last_error ~ '^[a-z_]{1,64}$'),
|
||
replays integer NOT NULL DEFAULT 0,
|
||
traceparent text CHECK (traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'),
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||
finished_at timestamptz,
|
||
UNIQUE (attachment_id, pipeline_version),
|
||
CHECK (attempt <= max_attempts),
|
||
CHECK ((status = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)),
|
||
CHECK ((status = 'failed') = (error_code IS NOT NULL))
|
||
);
|
||
CREATE INDEX attachment_jobs_leases ON attachment_jobs (lease_until) WHERE status = 'processing';
|
||
CREATE INDEX attachment_jobs_queued ON attachment_jobs (available_at) WHERE status = 'queued';
|
||
CREATE INDEX attachment_jobs_failed ON attachment_jobs (finished_at) WHERE status = 'failed';
|
||
|
||
-- Журнал (audit): claim, повторы, истечение аренды, дубли, replay. Только коды, без секретов и путей.
|
||
CREATE TABLE job_events (
|
||
id bigserial PRIMARY KEY,
|
||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id),
|
||
attempt integer NOT NULL,
|
||
event text NOT NULL CHECK (event ~ '^[a-z_]{1,64}$'),
|
||
detail text CHECK (char_length(detail) <= 200),
|
||
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 200),
|
||
created_at timestamptz NOT NULL DEFAULT now()
|
||
);
|
||
CREATE INDEX job_events_by_job ON job_events (job_id, id);
|
||
|
||
-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и задание.
|
||
-- dispatcher публикует строки с available_at <= now() и отмечает published_at после publisher confirm.
|
||
CREATE TABLE outbox (
|
||
id bigserial PRIMARY KEY,
|
||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id),
|
||
attempt integer NOT NULL,
|
||
payload jsonb NOT NULL,
|
||
available_at timestamptz NOT NULL DEFAULT now(),
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
published_at timestamptz,
|
||
publish_failures integer NOT NULL DEFAULT 0
|
||
);
|
||
CREATE INDEX outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL;
|
||
CREATE INDEX outbox_by_job ON outbox (job_id);
|
||
|
||
-- Задания для вложений, принятых в lab4 (миграция = их producer). JOB_MAX_ATTEMPTS по умолчанию 5.
|
||
INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, traceparent)
|
||
SELECT gen_random_uuid(), id, 1, 'queued', 1, 5,
|
||
'00-' || md5(random()::text || id::text) || '-' || substr(md5(id::text), 1, 16) || '-01'
|
||
FROM attachments WHERE state = 'queued';
|
||
|
||
INSERT INTO job_events (job_id, attempt, event, detail, actor)
|
||
SELECT job_id, 1, 'created', 'lab4_backfill', 'migration 005' FROM attachment_jobs;
|
||
|
||
INSERT INTO outbox (job_id, attempt, payload)
|
||
SELECT job_id, 1, jsonb_build_object(
|
||
'schema_version', 1,
|
||
'job_id', job_id,
|
||
'type', 'attachment.process.v1',
|
||
'attachment_id', attachment_id,
|
||
'pipeline_version', pipeline_version,
|
||
'attempt', 1,
|
||
'created_at', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'),
|
||
'traceparent', traceparent)
|
||
FROM attachment_jobs;
|