Files
mtusi2026lab5/migrations/0005_pipeline.sql
T
Alexey Bodanyuk a633949919 lab5: схема 0005 (jobs, outbox, попытки, варианты, аудит) и модули pipeline
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ,
published_version, jobs с lease и токеном попытки, job_attempts с заранее известными
ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs,
post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник.
pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем
процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной
транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения.
Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
2026-10-31 16:12:51 +03:00

261 lines
16 KiB
PL/PgSQL

-- 0005 (lab 5): asynchronous processing of attachments. Details: docs/PIPELINE.md.
--
-- Upload (API): INSERT attachment 'pending' (intent: the staging key is known before the PUT)
-- -> PUT original into the PRIVATE STAGING bucket
-- -> ONE transaction: attachment 'queued' + jobs row + outbox row -> 202.
-- Relay: outbox row -> publish to RabbitMQ with publisher confirm -> outbox.published_at.
-- Worker: claim job (lease + attempt token) -> variants into the final bucket under keys of
-- THIS attempt -> ONE transaction: variants + attachment 'ready' (fenced by the
-- lease token, never downgrading published_version) -> ack -> staging deleted.
--
-- PostgreSQL is the source of truth for the work: state, attempts, lease, backoff (available_at)
-- and the failed registry. The broker only delivers job ids (at least once).
-- ------------------------------------------------------------------ attachments
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
-- object_key = key of the ORIGINAL in the final bucket. NULL until the worker publishes it.
-- Keys of lab 4 ('o/<hex>') stay valid; lab 5 writes 'a/<attachment>/<attempt token>/original'.
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_format;
ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_format
CHECK (object_key ~ '^(o/[0-9a-f]{32}|a/[0-9a-f-]{36}/[0-9a-f]{32}/original)$');
ALTER TABLE attachments
-- Key of the original in the staging bucket ('s/<128 random bits>'), written by the API.
ADD COLUMN staging_key text COLLATE "C",
-- none (lab 4 row) | present | deleting (marked, S3 DELETE pending) | deleted
ADD COLUMN staging_state text NOT NULL DEFAULT 'none',
-- pipeline version whose variants are published (GET shows them); only ever grows.
ADD COLUMN published_version integer,
ADD COLUMN error_code text,
ADD COLUMN failed_at timestamptz,
ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key),
ADD CONSTRAINT attachments_staging_key_format CHECK (staging_key ~ '^s/[0-9a-f]{32}$'),
ADD CONSTRAINT attachments_staging_state CHECK (staging_state IN ('none', 'present', 'deleting', 'deleted')),
ADD CONSTRAINT attachments_staging_has_key CHECK (staging_state = 'none' OR staging_key IS NOT NULL),
ADD CONSTRAINT attachments_published_version CHECK (published_version >= 1),
ADD CONSTRAINT attachments_error_code CHECK (error_code ~ '^[a-z_]{1,64}$');
-- Lab 4 rows: a 'pending' intent of lab 4 never got its object confirmed and has no staging copy;
-- it becomes a failed upload (invisible: its id was never returned). A lab 4 'ready' row has its
-- original in the final bucket but no thumbnail yet: it is queued for pipeline 1 (below).
UPDATE attachments SET status = 'failed', error_code = 'upload_incomplete', failed_at = clock_timestamp()
WHERE status = 'pending';
UPDATE attachments SET status = 'queued' WHERE status = 'ready';
-- Structural guarantees (the database refuses a buggy cleanup or worker):
-- * work that is queued/processing always has a source: the published original or the staging
-- copy - the cleanup can never mark the staging of an accepted upload as deleting;
-- * ready = published original + published pipeline version; failed = error code.
ALTER TABLE attachments
ADD CONSTRAINT attachments_pending_staging CHECK (status <> 'pending' OR staging_state = 'present'),
ADD CONSTRAINT attachments_active_has_source
CHECK (status NOT IN ('queued', 'processing') OR object_key IS NOT NULL OR staging_state = 'present'),
ADD CONSTRAINT attachments_ready_published
CHECK (status <> 'ready' OR (object_key IS NOT NULL AND published_version IS NOT NULL)),
ADD CONSTRAINT attachments_failed_code
CHECK ((status = 'failed') = (error_code IS NOT NULL AND failed_at IS NOT NULL));
-- Janitor of the API: the lab 4 partial index on pending rows stays as is.
-- Cleanup of the worker: staging copies that may be deleted (ready: at once; failed: after retention).
CREATE INDEX attachments_staging_cleanup ON attachments (updated_at)
WHERE staging_state IN ('present', 'deleting') AND status IN ('ready', 'failed');
-- ------------------------------------------------------------------ jobs
-- One row = one logical piece of work: "build pipeline version V for attachment A".
-- job_id never changes on redelivery; `attempt` counts claims by workers (managed attempts).
CREATE TABLE jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id),
pipeline_version integer NOT NULL,
kind text NOT NULL, -- upload | replay | reprocess | migrate
replay_of uuid REFERENCES jobs (id),
state text NOT NULL DEFAULT 'queued',
attempt integer NOT NULL DEFAULT 0,
max_attempts integer NOT NULL,
available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- backoff: not before
lease_token uuid, -- token of the attempt that holds the job
lease_owner text,
lease_until timestamptz,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
finished_at timestamptz,
CONSTRAINT jobs_pipeline_version CHECK (pipeline_version >= 1),
CONSTRAINT jobs_kind CHECK (kind IN ('upload', 'replay', 'reprocess', 'migrate')),
CONSTRAINT jobs_replay CHECK ((kind = 'replay') = (replay_of IS NOT NULL)),
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed', 'superseded')),
CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts BETWEEN 1 AND 100 AND attempt <= max_attempts),
-- A lease exists exactly while the job is processing.
CONSTRAINT jobs_lease CHECK ((state = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)),
CONSTRAINT jobs_finished CHECK ((state IN ('succeeded', 'failed', 'superseded')) = (finished_at IS NOT NULL)),
CONSTRAINT jobs_failed_code CHECK (state <> 'failed' OR error_code IS NOT NULL),
CONSTRAINT jobs_error_code CHECK (error_code ~ '^[a-z_]{1,64}$')
);
-- At most ONE active job per effect set (attachment, pipeline version): a second upload job,
-- replay or reprocess of the same version cannot exist while one is queued/processing.
CREATE UNIQUE INDEX jobs_one_active ON jobs (attachment_id, pipeline_version) WHERE state IN ('queued', 'processing');
CREATE INDEX jobs_attachment ON jobs (attachment_id, created_at);
CREATE INDEX jobs_lease_expiry ON jobs (lease_until) WHERE state = 'processing'; -- reaper
CREATE INDEX jobs_queued ON jobs (updated_at) WHERE state = 'queued'; -- lost-message guard
CREATE INDEX jobs_failed ON jobs (finished_at) WHERE state = 'failed'; -- failed registry
-- One row per claim of a job by a worker. The token is part of every object key the attempt
-- writes, so the keys of an attempt are known without listing the bucket (planned_keys), and an
-- attempt that is no longer the lease holder can never publish (fencing by token).
CREATE TABLE job_attempts (
token uuid PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs (id),
attempt integer NOT NULL,
worker text NOT NULL,
source text NOT NULL, -- staging | original
planned_keys text[] NOT NULL,
outcome text NOT NULL DEFAULT 'running',
error_code text,
started_at timestamptz NOT NULL DEFAULT clock_timestamp(),
finished_at timestamptz,
objects_cleaned_at timestamptz,
CONSTRAINT job_attempts_number UNIQUE (job_id, attempt),
CONSTRAINT job_attempts_source CHECK (source IN ('staging', 'original')),
CONSTRAINT job_attempts_outcome
CHECK (outcome IN ('running', 'succeeded', 'retry', 'failed', 'abandoned', 'superseded')),
CONSTRAINT job_attempts_finished CHECK ((outcome = 'running') = (finished_at IS NULL)),
-- Objects of a successful attempt are published: they are never cleaned.
CONSTRAINT job_attempts_cleaned CHECK (objects_cleaned_at IS NULL OR outcome NOT IN ('running', 'succeeded'))
);
CREATE INDEX job_attempts_cleanup ON job_attempts (finished_at)
WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') AND objects_cleaned_at IS NULL;
-- Published variants. The primary key is the idempotency key of the EFFECT from the contract:
-- (attachment_id, pipeline_version, variant) exists at most once, whatever the deliveries.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id),
pipeline_version integer NOT NULL,
name text NOT NULL,
object_key text COLLATE "C" NOT NULL,
content_type text NOT NULL,
size_bytes integer NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 bytea NOT NULL,
attempt_token uuid NOT NULL REFERENCES job_attempts (token),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT attachment_variants_key UNIQUE (object_key),
CONSTRAINT attachment_variants_key_format CHECK (object_key ~ '^a/[0-9a-f-]{36}/[0-9a-f]{32}/(thumbnail|crop|optimized)$'),
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp')),
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1 AND width >= 1 AND height >= 1),
CONSTRAINT attachment_variants_sha256 CHECK (octet_length(sha256) = 32)
);
-- Transactional outbox: written in the SAME transaction as the job (upload, retry, replay,
-- reprocess, lease expiry). The relay publishes due rows and marks them; a crash after the
-- publish but before the mark publishes again (a duplicate the worker ignores).
CREATE TABLE outbox (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs (id),
attempt integer NOT NULL, -- the attempt this delivery is meant for
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
publish_failures integer NOT NULL DEFAULT 0,
CONSTRAINT outbox_attempt CHECK (attempt >= 1)
);
CREATE INDEX outbox_due ON outbox (available_at, id) WHERE published_at IS NULL;
CREATE INDEX outbox_job ON outbox (job_id);
CREATE INDEX outbox_published ON outbox (published_at) WHERE published_at IS NOT NULL; -- purge
-- Audit of every transition (no secrets, no file names): replay/reprocess requests with the
-- operator and reason, claims, retries with their delay, lease expiries, results.
CREATE TABLE job_events (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs (id),
at timestamptz NOT NULL DEFAULT clock_timestamp(),
event text NOT NULL,
attempt integer,
actor text NOT NULL,
error_code text,
detail jsonb NOT NULL DEFAULT '{}'
);
CREATE INDEX job_events_job ON job_events (job_id, id);
-- Failed registry for the operator (jobctl list-failed).
CREATE VIEW failed_jobs AS
SELECT j.id AS job_id, j.attachment_id, j.pipeline_version, j.kind, j.attempt, j.max_attempts, j.error_code,
j.finished_at, a.status AS attachment_status, a.staging_state
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.state = 'failed';
-- Lab 4 'ready' rows (now queued): a job for pipeline 1 from their published original.
WITH queued AS (
SELECT a.id, gen_random_uuid() AS job_id FROM attachments a WHERE a.status = 'queued'
), created AS (
INSERT INTO jobs (id, attachment_id, pipeline_version, kind, max_attempts)
SELECT q.job_id, q.id, 1, 'migrate', 5 FROM queued q RETURNING id
)
INSERT INTO outbox (job_id, attempt) SELECT c.id, 1 FROM created c;
-- post_message_v3 = post_message_v2 (0004) with the lab 5 attachment states: queued, processing
-- and ready attachments of this chat may be attached; failed (or quarantined) -> 409; pending,
-- foreign or unknown -> 404. Everything else (key claim, chat clock, one transaction) unchanged.
CREATE FUNCTION post_message_v3(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
LANGUAGE plpgsql VOLATILE AS $$
DECLARE
v_created timestamptz;
v_found integer;
v_usable integer;
v_check text := 'ok';
BEGIN
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
IF NOT FOUND THEN
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
RETURN;
END IF;
IF cardinality(p_attachments) > 0 THEN
-- FOR SHARE: a concurrent transition to failed/quarantine waits until this message commits.
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') INTO v_found, v_usable
FROM (SELECT a.lost_at, a.status FROM attachments a
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status <> 'pending'
FOR SHARE) a;
IF v_found <> cardinality(p_attachments) THEN
v_check := 'attachment_not_found';
ELSIF v_usable <> v_found THEN
v_check := 'attachment_conflict';
END IF;
END IF;
IF v_check = 'ok' THEN
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, p_message)
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
IF FOUND THEN
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
INSERT INTO message_attachments (message_id, position, attachment_id)
SELECT p_message, (a.ord - 1)::smallint, a.id
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
RETURN;
END IF;
END IF;
RETURN QUERY
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
m.attachment_ids
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
IF NOT FOUND THEN
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
END IF;
END
$$;