forked from mtusi2026/mtusi2026lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ, published_version, jobs с lease и токеном попытки, job_attempts с заранее известными ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs, post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник. pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения. Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
261 lines
16 KiB
PL/PgSQL
261 lines
16 KiB
PL/PgSQL
-- 0005 (lab 5): asynchronous processing of attachments. Details: docs/PIPELINE.md.
|
|
--
|
|
-- Upload (API): INSERT attachment 'pending' (intent: the staging key is known before the PUT)
|
|
-- -> PUT original into the PRIVATE STAGING bucket
|
|
-- -> ONE transaction: attachment 'queued' + jobs row + outbox row -> 202.
|
|
-- Relay: outbox row -> publish to RabbitMQ with publisher confirm -> outbox.published_at.
|
|
-- Worker: claim job (lease + attempt token) -> variants into the final bucket under keys of
|
|
-- THIS attempt -> ONE transaction: variants + attachment 'ready' (fenced by the
|
|
-- lease token, never downgrading published_version) -> ack -> staging deleted.
|
|
--
|
|
-- PostgreSQL is the source of truth for the work: state, attempts, lease, backoff (available_at)
|
|
-- and the failed registry. The broker only delivers job ids (at least once).
|
|
|
|
-- ------------------------------------------------------------------ attachments
|
|
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
|
ALTER TABLE attachments ADD CONSTRAINT attachments_status
|
|
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
|
|
-- object_key = key of the ORIGINAL in the final bucket. NULL until the worker publishes it.
|
|
-- Keys of lab 4 ('o/<hex>') stay valid; lab 5 writes 'a/<attachment>/<attempt token>/original'.
|
|
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
|
|
ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_format;
|
|
ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_format
|
|
CHECK (object_key ~ '^(o/[0-9a-f]{32}|a/[0-9a-f-]{36}/[0-9a-f]{32}/original)$');
|
|
ALTER TABLE attachments
|
|
-- Key of the original in the staging bucket ('s/<128 random bits>'), written by the API.
|
|
ADD COLUMN staging_key text COLLATE "C",
|
|
-- none (lab 4 row) | present | deleting (marked, S3 DELETE pending) | deleted
|
|
ADD COLUMN staging_state text NOT NULL DEFAULT 'none',
|
|
-- pipeline version whose variants are published (GET shows them); only ever grows.
|
|
ADD COLUMN published_version integer,
|
|
ADD COLUMN error_code text,
|
|
ADD COLUMN failed_at timestamptz,
|
|
ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key),
|
|
ADD CONSTRAINT attachments_staging_key_format CHECK (staging_key ~ '^s/[0-9a-f]{32}$'),
|
|
ADD CONSTRAINT attachments_staging_state CHECK (staging_state IN ('none', 'present', 'deleting', 'deleted')),
|
|
ADD CONSTRAINT attachments_staging_has_key CHECK (staging_state = 'none' OR staging_key IS NOT NULL),
|
|
ADD CONSTRAINT attachments_published_version CHECK (published_version >= 1),
|
|
ADD CONSTRAINT attachments_error_code CHECK (error_code ~ '^[a-z_]{1,64}$');
|
|
|
|
-- Lab 4 rows: a 'pending' intent of lab 4 never got its object confirmed and has no staging copy;
|
|
-- it becomes a failed upload (invisible: its id was never returned). A lab 4 'ready' row has its
|
|
-- original in the final bucket but no thumbnail yet: it is queued for pipeline 1 (below).
|
|
UPDATE attachments SET status = 'failed', error_code = 'upload_incomplete', failed_at = clock_timestamp()
|
|
WHERE status = 'pending';
|
|
UPDATE attachments SET status = 'queued' WHERE status = 'ready';
|
|
|
|
-- Structural guarantees (the database refuses a buggy cleanup or worker):
|
|
-- * work that is queued/processing always has a source: the published original or the staging
|
|
-- copy - the cleanup can never mark the staging of an accepted upload as deleting;
|
|
-- * ready = published original + published pipeline version; failed = error code.
|
|
ALTER TABLE attachments
|
|
ADD CONSTRAINT attachments_pending_staging CHECK (status <> 'pending' OR staging_state = 'present'),
|
|
ADD CONSTRAINT attachments_active_has_source
|
|
CHECK (status NOT IN ('queued', 'processing') OR object_key IS NOT NULL OR staging_state = 'present'),
|
|
ADD CONSTRAINT attachments_ready_published
|
|
CHECK (status <> 'ready' OR (object_key IS NOT NULL AND published_version IS NOT NULL)),
|
|
ADD CONSTRAINT attachments_failed_code
|
|
CHECK ((status = 'failed') = (error_code IS NOT NULL AND failed_at IS NOT NULL));
|
|
-- Janitor of the API: the lab 4 partial index on pending rows stays as is.
|
|
-- Cleanup of the worker: staging copies that may be deleted (ready: at once; failed: after retention).
|
|
CREATE INDEX attachments_staging_cleanup ON attachments (updated_at)
|
|
WHERE staging_state IN ('present', 'deleting') AND status IN ('ready', 'failed');
|
|
|
|
-- ------------------------------------------------------------------ jobs
|
|
-- One row = one logical piece of work: "build pipeline version V for attachment A".
|
|
-- job_id never changes on redelivery; `attempt` counts claims by workers (managed attempts).
|
|
CREATE TABLE jobs (
|
|
id uuid PRIMARY KEY,
|
|
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
|
pipeline_version integer NOT NULL,
|
|
kind text NOT NULL, -- upload | replay | reprocess | migrate
|
|
replay_of uuid REFERENCES jobs (id),
|
|
state text NOT NULL DEFAULT 'queued',
|
|
attempt integer NOT NULL DEFAULT 0,
|
|
max_attempts integer NOT NULL,
|
|
available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- backoff: not before
|
|
lease_token uuid, -- token of the attempt that holds the job
|
|
lease_owner text,
|
|
lease_until timestamptz,
|
|
error_code text,
|
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
finished_at timestamptz,
|
|
CONSTRAINT jobs_pipeline_version CHECK (pipeline_version >= 1),
|
|
CONSTRAINT jobs_kind CHECK (kind IN ('upload', 'replay', 'reprocess', 'migrate')),
|
|
CONSTRAINT jobs_replay CHECK ((kind = 'replay') = (replay_of IS NOT NULL)),
|
|
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed', 'superseded')),
|
|
CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts BETWEEN 1 AND 100 AND attempt <= max_attempts),
|
|
-- A lease exists exactly while the job is processing.
|
|
CONSTRAINT jobs_lease CHECK ((state = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)),
|
|
CONSTRAINT jobs_finished CHECK ((state IN ('succeeded', 'failed', 'superseded')) = (finished_at IS NOT NULL)),
|
|
CONSTRAINT jobs_failed_code CHECK (state <> 'failed' OR error_code IS NOT NULL),
|
|
CONSTRAINT jobs_error_code CHECK (error_code ~ '^[a-z_]{1,64}$')
|
|
);
|
|
-- At most ONE active job per effect set (attachment, pipeline version): a second upload job,
|
|
-- replay or reprocess of the same version cannot exist while one is queued/processing.
|
|
CREATE UNIQUE INDEX jobs_one_active ON jobs (attachment_id, pipeline_version) WHERE state IN ('queued', 'processing');
|
|
CREATE INDEX jobs_attachment ON jobs (attachment_id, created_at);
|
|
CREATE INDEX jobs_lease_expiry ON jobs (lease_until) WHERE state = 'processing'; -- reaper
|
|
CREATE INDEX jobs_queued ON jobs (updated_at) WHERE state = 'queued'; -- lost-message guard
|
|
CREATE INDEX jobs_failed ON jobs (finished_at) WHERE state = 'failed'; -- failed registry
|
|
|
|
-- One row per claim of a job by a worker. The token is part of every object key the attempt
|
|
-- writes, so the keys of an attempt are known without listing the bucket (planned_keys), and an
|
|
-- attempt that is no longer the lease holder can never publish (fencing by token).
|
|
CREATE TABLE job_attempts (
|
|
token uuid PRIMARY KEY,
|
|
job_id uuid NOT NULL REFERENCES jobs (id),
|
|
attempt integer NOT NULL,
|
|
worker text NOT NULL,
|
|
source text NOT NULL, -- staging | original
|
|
planned_keys text[] NOT NULL,
|
|
outcome text NOT NULL DEFAULT 'running',
|
|
error_code text,
|
|
started_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
finished_at timestamptz,
|
|
objects_cleaned_at timestamptz,
|
|
CONSTRAINT job_attempts_number UNIQUE (job_id, attempt),
|
|
CONSTRAINT job_attempts_source CHECK (source IN ('staging', 'original')),
|
|
CONSTRAINT job_attempts_outcome
|
|
CHECK (outcome IN ('running', 'succeeded', 'retry', 'failed', 'abandoned', 'superseded')),
|
|
CONSTRAINT job_attempts_finished CHECK ((outcome = 'running') = (finished_at IS NULL)),
|
|
-- Objects of a successful attempt are published: they are never cleaned.
|
|
CONSTRAINT job_attempts_cleaned CHECK (objects_cleaned_at IS NULL OR outcome NOT IN ('running', 'succeeded'))
|
|
);
|
|
CREATE INDEX job_attempts_cleanup ON job_attempts (finished_at)
|
|
WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') AND objects_cleaned_at IS NULL;
|
|
|
|
-- Published variants. The primary key is the idempotency key of the EFFECT from the contract:
|
|
-- (attachment_id, pipeline_version, variant) exists at most once, whatever the deliveries.
|
|
CREATE TABLE attachment_variants (
|
|
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
|
pipeline_version integer NOT NULL,
|
|
name text NOT NULL,
|
|
object_key text COLLATE "C" NOT NULL,
|
|
content_type text NOT NULL,
|
|
size_bytes integer NOT NULL,
|
|
width integer NOT NULL,
|
|
height integer NOT NULL,
|
|
sha256 bytea NOT NULL,
|
|
attempt_token uuid NOT NULL REFERENCES job_attempts (token),
|
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
PRIMARY KEY (attachment_id, pipeline_version, name),
|
|
CONSTRAINT attachment_variants_key UNIQUE (object_key),
|
|
CONSTRAINT attachment_variants_key_format CHECK (object_key ~ '^a/[0-9a-f-]{36}/[0-9a-f]{32}/(thumbnail|crop|optimized)$'),
|
|
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
|
CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp')),
|
|
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1 AND width >= 1 AND height >= 1),
|
|
CONSTRAINT attachment_variants_sha256 CHECK (octet_length(sha256) = 32)
|
|
);
|
|
|
|
-- Transactional outbox: written in the SAME transaction as the job (upload, retry, replay,
|
|
-- reprocess, lease expiry). The relay publishes due rows and marks them; a crash after the
|
|
-- publish but before the mark publishes again (a duplicate the worker ignores).
|
|
CREATE TABLE outbox (
|
|
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
job_id uuid NOT NULL REFERENCES jobs (id),
|
|
attempt integer NOT NULL, -- the attempt this delivery is meant for
|
|
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
published_at timestamptz,
|
|
publish_failures integer NOT NULL DEFAULT 0,
|
|
CONSTRAINT outbox_attempt CHECK (attempt >= 1)
|
|
);
|
|
CREATE INDEX outbox_due ON outbox (available_at, id) WHERE published_at IS NULL;
|
|
CREATE INDEX outbox_job ON outbox (job_id);
|
|
CREATE INDEX outbox_published ON outbox (published_at) WHERE published_at IS NOT NULL; -- purge
|
|
|
|
-- Audit of every transition (no secrets, no file names): replay/reprocess requests with the
|
|
-- operator and reason, claims, retries with their delay, lease expiries, results.
|
|
CREATE TABLE job_events (
|
|
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
job_id uuid NOT NULL REFERENCES jobs (id),
|
|
at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
event text NOT NULL,
|
|
attempt integer,
|
|
actor text NOT NULL,
|
|
error_code text,
|
|
detail jsonb NOT NULL DEFAULT '{}'
|
|
);
|
|
CREATE INDEX job_events_job ON job_events (job_id, id);
|
|
|
|
-- Failed registry for the operator (jobctl list-failed).
|
|
CREATE VIEW failed_jobs AS
|
|
SELECT j.id AS job_id, j.attachment_id, j.pipeline_version, j.kind, j.attempt, j.max_attempts, j.error_code,
|
|
j.finished_at, a.status AS attachment_status, a.staging_state
|
|
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
|
WHERE j.state = 'failed';
|
|
|
|
-- Lab 4 'ready' rows (now queued): a job for pipeline 1 from their published original.
|
|
WITH queued AS (
|
|
SELECT a.id, gen_random_uuid() AS job_id FROM attachments a WHERE a.status = 'queued'
|
|
), created AS (
|
|
INSERT INTO jobs (id, attachment_id, pipeline_version, kind, max_attempts)
|
|
SELECT q.job_id, q.id, 1, 'migrate', 5 FROM queued q RETURNING id
|
|
)
|
|
INSERT INTO outbox (job_id, attempt) SELECT c.id, 1 FROM created c;
|
|
|
|
-- post_message_v3 = post_message_v2 (0004) with the lab 5 attachment states: queued, processing
|
|
-- and ready attachments of this chat may be attached; failed (or quarantined) -> 409; pending,
|
|
-- foreign or unknown -> 404. Everything else (key claim, chat clock, one transaction) unchanged.
|
|
CREATE FUNCTION post_message_v3(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
|
|
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
|
|
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
|
|
LANGUAGE plpgsql VOLATILE AS $$
|
|
DECLARE
|
|
v_created timestamptz;
|
|
v_found integer;
|
|
v_usable integer;
|
|
v_check text := 'ok';
|
|
BEGIN
|
|
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
|
|
IF NOT FOUND THEN
|
|
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
|
RETURN;
|
|
END IF;
|
|
|
|
IF cardinality(p_attachments) > 0 THEN
|
|
-- FOR SHARE: a concurrent transition to failed/quarantine waits until this message commits.
|
|
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') INTO v_found, v_usable
|
|
FROM (SELECT a.lost_at, a.status FROM attachments a
|
|
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status <> 'pending'
|
|
FOR SHARE) a;
|
|
IF v_found <> cardinality(p_attachments) THEN
|
|
v_check := 'attachment_not_found';
|
|
ELSIF v_usable <> v_found THEN
|
|
v_check := 'attachment_conflict';
|
|
END IF;
|
|
END IF;
|
|
|
|
IF v_check = 'ok' THEN
|
|
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
|
|
VALUES (p_user, p_chat, p_key, p_hash, p_message)
|
|
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
|
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
|
|
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
|
|
IF FOUND THEN
|
|
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
|
|
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
|
|
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
|
|
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
|
|
INSERT INTO message_attachments (message_id, position, attachment_id)
|
|
SELECT p_message, (a.ord - 1)::smallint, a.id
|
|
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
|
|
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
|
|
RETURN;
|
|
END IF;
|
|
END IF;
|
|
|
|
RETURN QUERY
|
|
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
|
|
m.attachment_ids
|
|
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
|
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
|
|
IF NOT FOUND THEN
|
|
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
|
|
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
|
END IF;
|
|
END
|
|
$$;
|