-- 0005 (lab 5): asynchronous processing of attachments. Details: docs/PIPELINE.md. -- -- Upload (API): INSERT attachment 'pending' (intent: the staging key is known before the PUT) -- -> PUT original into the PRIVATE STAGING bucket -- -> ONE transaction: attachment 'queued' + jobs row + outbox row -> 202. -- Relay: outbox row -> publish to RabbitMQ with publisher confirm -> outbox.published_at. -- Worker: claim job (lease + attempt token) -> variants into the final bucket under keys of -- THIS attempt -> ONE transaction: variants + attachment 'ready' (fenced by the -- lease token, never downgrading published_version) -> ack -> staging deleted. -- -- PostgreSQL is the source of truth for the work: state, attempts, lease, backoff (available_at) -- and the failed registry. The broker only delivers job ids (at least once). -- ------------------------------------------------------------------ attachments ALTER TABLE attachments DROP CONSTRAINT attachments_status; ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed')); -- object_key = key of the ORIGINAL in the final bucket. NULL until the worker publishes it. -- Keys of lab 4 ('o/') stay valid; lab 5 writes 'a///original'. ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL; ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_format; ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_format CHECK (object_key ~ '^(o/[0-9a-f]{32}|a/[0-9a-f-]{36}/[0-9a-f]{32}/original)$'); ALTER TABLE attachments -- Key of the original in the staging bucket ('s/<128 random bits>'), written by the API. ADD COLUMN staging_key text COLLATE "C", -- none (lab 4 row) | present | deleting (marked, S3 DELETE pending) | deleted ADD COLUMN staging_state text NOT NULL DEFAULT 'none', -- pipeline version whose variants are published (GET shows them); only ever grows. ADD COLUMN published_version integer, ADD COLUMN error_code text, ADD COLUMN failed_at timestamptz, ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key), ADD CONSTRAINT attachments_staging_key_format CHECK (staging_key ~ '^s/[0-9a-f]{32}$'), ADD CONSTRAINT attachments_staging_state CHECK (staging_state IN ('none', 'present', 'deleting', 'deleted')), ADD CONSTRAINT attachments_staging_has_key CHECK (staging_state = 'none' OR staging_key IS NOT NULL), ADD CONSTRAINT attachments_published_version CHECK (published_version >= 1), ADD CONSTRAINT attachments_error_code CHECK (error_code ~ '^[a-z_]{1,64}$'); -- Lab 4 rows: a 'pending' intent of lab 4 never got its object confirmed and has no staging copy; -- it becomes a failed upload (invisible: its id was never returned). A lab 4 'ready' row has its -- original in the final bucket but no thumbnail yet: it is queued for pipeline 1 (below). UPDATE attachments SET status = 'failed', error_code = 'upload_incomplete', failed_at = clock_timestamp() WHERE status = 'pending'; UPDATE attachments SET status = 'queued' WHERE status = 'ready'; -- Structural guarantees (the database refuses a buggy cleanup or worker): -- * work that is queued/processing always has a source: the published original or the staging -- copy - the cleanup can never mark the staging of an accepted upload as deleting; -- * ready = published original + published pipeline version; failed = error code. ALTER TABLE attachments ADD CONSTRAINT attachments_pending_staging CHECK (status <> 'pending' OR staging_state = 'present'), ADD CONSTRAINT attachments_active_has_source CHECK (status NOT IN ('queued', 'processing') OR object_key IS NOT NULL OR staging_state = 'present'), ADD CONSTRAINT attachments_ready_published CHECK (status <> 'ready' OR (object_key IS NOT NULL AND published_version IS NOT NULL)), ADD CONSTRAINT attachments_failed_code CHECK ((status = 'failed') = (error_code IS NOT NULL AND failed_at IS NOT NULL)); -- Janitor of the API: the lab 4 partial index on pending rows stays as is. -- Cleanup of the worker: staging copies that may be deleted (ready: at once; failed: after retention). CREATE INDEX attachments_staging_cleanup ON attachments (updated_at) WHERE staging_state IN ('present', 'deleting') AND status IN ('ready', 'failed'); -- ------------------------------------------------------------------ jobs -- One row = one logical piece of work: "build pipeline version V for attachment A". -- job_id never changes on redelivery; `attempt` counts claims by workers (managed attempts). CREATE TABLE jobs ( id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments (id), pipeline_version integer NOT NULL, kind text NOT NULL, -- upload | replay | reprocess | migrate replay_of uuid REFERENCES jobs (id), state text NOT NULL DEFAULT 'queued', attempt integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL, available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- backoff: not before lease_token uuid, -- token of the attempt that holds the job lease_owner text, lease_until timestamptz, error_code text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), finished_at timestamptz, CONSTRAINT jobs_pipeline_version CHECK (pipeline_version >= 1), CONSTRAINT jobs_kind CHECK (kind IN ('upload', 'replay', 'reprocess', 'migrate')), CONSTRAINT jobs_replay CHECK ((kind = 'replay') = (replay_of IS NOT NULL)), CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed', 'superseded')), CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts BETWEEN 1 AND 100 AND attempt <= max_attempts), -- A lease exists exactly while the job is processing. CONSTRAINT jobs_lease CHECK ((state = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)), CONSTRAINT jobs_finished CHECK ((state IN ('succeeded', 'failed', 'superseded')) = (finished_at IS NOT NULL)), CONSTRAINT jobs_failed_code CHECK (state <> 'failed' OR error_code IS NOT NULL), CONSTRAINT jobs_error_code CHECK (error_code ~ '^[a-z_]{1,64}$') ); -- At most ONE active job per effect set (attachment, pipeline version): a second upload job, -- replay or reprocess of the same version cannot exist while one is queued/processing. CREATE UNIQUE INDEX jobs_one_active ON jobs (attachment_id, pipeline_version) WHERE state IN ('queued', 'processing'); CREATE INDEX jobs_attachment ON jobs (attachment_id, created_at); CREATE INDEX jobs_lease_expiry ON jobs (lease_until) WHERE state = 'processing'; -- reaper CREATE INDEX jobs_queued ON jobs (updated_at) WHERE state = 'queued'; -- lost-message guard CREATE INDEX jobs_failed ON jobs (finished_at) WHERE state = 'failed'; -- failed registry -- One row per claim of a job by a worker. The token is part of every object key the attempt -- writes, so the keys of an attempt are known without listing the bucket (planned_keys), and an -- attempt that is no longer the lease holder can never publish (fencing by token). CREATE TABLE job_attempts ( token uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs (id), attempt integer NOT NULL, worker text NOT NULL, source text NOT NULL, -- staging | original planned_keys text[] NOT NULL, outcome text NOT NULL DEFAULT 'running', error_code text, started_at timestamptz NOT NULL DEFAULT clock_timestamp(), finished_at timestamptz, objects_cleaned_at timestamptz, CONSTRAINT job_attempts_number UNIQUE (job_id, attempt), CONSTRAINT job_attempts_source CHECK (source IN ('staging', 'original')), CONSTRAINT job_attempts_outcome CHECK (outcome IN ('running', 'succeeded', 'retry', 'failed', 'abandoned', 'superseded')), CONSTRAINT job_attempts_finished CHECK ((outcome = 'running') = (finished_at IS NULL)), -- Objects of a successful attempt are published: they are never cleaned. CONSTRAINT job_attempts_cleaned CHECK (objects_cleaned_at IS NULL OR outcome NOT IN ('running', 'succeeded')) ); CREATE INDEX job_attempts_cleanup ON job_attempts (finished_at) WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') AND objects_cleaned_at IS NULL; -- Published variants. The primary key is the idempotency key of the EFFECT from the contract: -- (attachment_id, pipeline_version, variant) exists at most once, whatever the deliveries. CREATE TABLE attachment_variants ( attachment_id uuid NOT NULL REFERENCES attachments (id), pipeline_version integer NOT NULL, name text NOT NULL, object_key text COLLATE "C" NOT NULL, content_type text NOT NULL, size_bytes integer NOT NULL, width integer NOT NULL, height integer NOT NULL, sha256 bytea NOT NULL, attempt_token uuid NOT NULL REFERENCES job_attempts (token), created_at timestamptz NOT NULL DEFAULT clock_timestamp(), PRIMARY KEY (attachment_id, pipeline_version, name), CONSTRAINT attachment_variants_key UNIQUE (object_key), CONSTRAINT attachment_variants_key_format CHECK (object_key ~ '^a/[0-9a-f-]{36}/[0-9a-f]{32}/(thumbnail|crop|optimized)$'), CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp')), CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1 AND width >= 1 AND height >= 1), CONSTRAINT attachment_variants_sha256 CHECK (octet_length(sha256) = 32) ); -- Transactional outbox: written in the SAME transaction as the job (upload, retry, replay, -- reprocess, lease expiry). The relay publishes due rows and marks them; a crash after the -- publish but before the mark publishes again (a duplicate the worker ignores). CREATE TABLE outbox ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs (id), attempt integer NOT NULL, -- the attempt this delivery is meant for available_at timestamptz NOT NULL DEFAULT clock_timestamp(), created_at timestamptz NOT NULL DEFAULT clock_timestamp(), published_at timestamptz, publish_failures integer NOT NULL DEFAULT 0, CONSTRAINT outbox_attempt CHECK (attempt >= 1) ); CREATE INDEX outbox_due ON outbox (available_at, id) WHERE published_at IS NULL; CREATE INDEX outbox_job ON outbox (job_id); CREATE INDEX outbox_published ON outbox (published_at) WHERE published_at IS NOT NULL; -- purge -- Audit of every transition (no secrets, no file names): replay/reprocess requests with the -- operator and reason, claims, retries with their delay, lease expiries, results. CREATE TABLE job_events ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs (id), at timestamptz NOT NULL DEFAULT clock_timestamp(), event text NOT NULL, attempt integer, actor text NOT NULL, error_code text, detail jsonb NOT NULL DEFAULT '{}' ); CREATE INDEX job_events_job ON job_events (job_id, id); -- Failed registry for the operator (jobctl list-failed). CREATE VIEW failed_jobs AS SELECT j.id AS job_id, j.attachment_id, j.pipeline_version, j.kind, j.attempt, j.max_attempts, j.error_code, j.finished_at, a.status AS attachment_status, a.staging_state FROM jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.state = 'failed'; -- Lab 4 'ready' rows (now queued): a job for pipeline 1 from their published original. WITH queued AS ( SELECT a.id, gen_random_uuid() AS job_id FROM attachments a WHERE a.status = 'queued' ), created AS ( INSERT INTO jobs (id, attachment_id, pipeline_version, kind, max_attempts) SELECT q.job_id, q.id, 1, 'migrate', 5 FROM queued q RETURNING id ) INSERT INTO outbox (job_id, attempt) SELECT c.id, 1 FROM created c; -- post_message_v3 = post_message_v2 (0004) with the lab 5 attachment states: queued, processing -- and ready attachments of this chat may be attached; failed (or quarantined) -> 409; pending, -- foreign or unknown -> 404. Everything else (key claim, chat clock, one transaction) unchanged. CREATE FUNCTION post_message_v3(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid, p_text bytea, p_ttl_seconds integer, p_attachments uuid[]) RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[]) LANGUAGE plpgsql VOLATILE AS $$ DECLARE v_created timestamptz; v_found integer; v_usable integer; v_check text := 'ok'; BEGIN PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user; IF NOT FOUND THEN RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; RETURN; END IF; IF cardinality(p_attachments) > 0 THEN -- FOR SHARE: a concurrent transition to failed/quarantine waits until this message commits. SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') INTO v_found, v_usable FROM (SELECT a.lost_at, a.status FROM attachments a WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status <> 'pending' FOR SHARE) a; IF v_found <> cardinality(p_attachments) THEN v_check := 'attachment_not_found'; ELSIF v_usable <> v_found THEN v_check := 'attachment_conflict'; END IF; END IF; IF v_check = 'ok' THEN INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id) VALUES (p_user, p_chat, p_key, p_hash, p_message) ON CONFLICT (user_id, chat_id, key) DO UPDATE SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds); IF FOUND THEN UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond') WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created; INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments); INSERT INTO message_attachments (message_id, position, attachment_id) SELECT p_message, (a.ord - 1)::smallint, a.id FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord); RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments; RETURN; END IF; END IF; RETURN QUERY SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k JOIN messages m ON m.id = k.message_id WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key; IF NOT FOUND THEN RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; END IF; END $$;