forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
DROP TABLE IF EXISTS attachment_jobs;
|
||||
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_variants_array;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_pipeline_version;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
|
||||
|
||||
-- Collapse asynchronous states back to lab4's single in-flight state. Ready
|
||||
-- and failed rows keep their meaning; queued/processing work is abandoned.
|
||||
UPDATE attachments
|
||||
SET status = 'pending', error_code = NULL
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
ALTER TABLE attachments
|
||||
DROP COLUMN IF EXISTS variants,
|
||||
DROP COLUMN IF EXISTS pipeline_version,
|
||||
DROP COLUMN IF EXISTS staging_key,
|
||||
DROP COLUMN IF EXISTS ready_at;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('pending', 'ready', 'failed')),
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status = 'pending' AND error_code IS NULL) OR
|
||||
(status = 'ready' AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
);
|
||||
@@ -0,0 +1,76 @@
|
||||
-- Lab5 turns attachment processing into a durable asynchronous job.
|
||||
-- The API stages bytes in S3, then in ONE transaction writes the attachment
|
||||
-- snapshot and its outbox/job row. A separate publisher moves the intent to
|
||||
-- RabbitMQ, so a broker outage cannot lose an accepted upload.
|
||||
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
|
||||
-- Lab4 "pending" meant "object PUT in progress". Nothing can finish it now,
|
||||
-- because lab5 acceptors write "queued" plus a job. Preserve the row for the
|
||||
-- operator instead of silently deleting it.
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = 'legacy_interrupted', updated_at = now()
|
||||
WHERE status = 'pending';
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN IF NOT EXISTS staging_key text,
|
||||
ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS variants jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS ready_at timestamptz;
|
||||
|
||||
UPDATE attachments SET ready_at = updated_at WHERE status = 'ready' AND ready_at IS NULL;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
),
|
||||
ADD CONSTRAINT attachments_variants_array
|
||||
CHECK (jsonb_typeof(variants) = 'array'),
|
||||
ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1);
|
||||
|
||||
-- One job per logical attachment effect. published_at IS NULL is the outbox
|
||||
-- marker: a committed intent that has not received a publisher confirm.
|
||||
CREATE TABLE IF NOT EXISTS attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
type text NOT NULL,
|
||||
pipeline_version integer NOT NULL,
|
||||
attempt integer NOT NULL DEFAULT 0,
|
||||
max_attempts integer NOT NULL,
|
||||
status text NOT NULL,
|
||||
error_code text,
|
||||
available_at timestamptz NOT NULL,
|
||||
published_at timestamptz,
|
||||
lease_until timestamptz,
|
||||
processing_started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
replay_count integer NOT NULL DEFAULT 0,
|
||||
last_replay_at timestamptz,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
CONSTRAINT attachment_jobs_status
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_attempts CHECK (attempt >= 0 AND max_attempts >= 1),
|
||||
CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT attachment_jobs_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
),
|
||||
CONSTRAINT attachment_jobs_finished CHECK (
|
||||
(status IN ('queued', 'processing') AND finished_at IS NULL) OR
|
||||
(status IN ('ready', 'failed') AND finished_at IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachment_jobs_outbox_idx
|
||||
ON attachment_jobs (available_at, created_at)
|
||||
WHERE published_at IS NULL AND status = 'queued';
|
||||
CREATE INDEX IF NOT EXISTS attachment_jobs_lease_idx
|
||||
ON attachment_jobs (lease_until)
|
||||
WHERE status = 'processing';
|
||||
CREATE INDEX IF NOT EXISTS attachment_jobs_status_idx
|
||||
ON attachment_jobs (status, created_at);
|
||||
@@ -0,0 +1,24 @@
|
||||
-- Returning to the stricter 0004 shape requires clearing retry reasons first.
|
||||
UPDATE attachment_jobs
|
||||
SET error_code = NULL
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
UPDATE attachments
|
||||
SET error_code = NULL
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
);
|
||||
|
||||
ALTER TABLE attachment_jobs
|
||||
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD CONSTRAINT attachment_jobs_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
);
|
||||
@@ -0,0 +1,23 @@
|
||||
-- ScheduleRetry records the transient reason on the job while it waits for the
|
||||
-- next attempt. The original 0004 constraint only allowed error_code on failed
|
||||
-- rows, so a jittered retry update was rejected and the broker redelivery was
|
||||
-- then acknowledged as "busy", leaving the job processing until lease expiry.
|
||||
-- Ready stays clean; failed stays explainable; queued/processing may carry the
|
||||
-- most recent transient reason.
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status = 'ready' AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL) OR
|
||||
(status IN ('queued', 'processing'))
|
||||
);
|
||||
|
||||
ALTER TABLE attachment_jobs
|
||||
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD CONSTRAINT attachment_jobs_error_state CHECK (
|
||||
(status = 'ready' AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL) OR
|
||||
(status IN ('queued', 'processing'))
|
||||
);
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
// latestMigrationVersion is the only value to bump when a migration is added.
|
||||
// The rest of the test derives filenames and ordering from the directory.
|
||||
const latestMigrationVersion = 3
|
||||
const latestMigrationVersion = 5
|
||||
|
||||
var migrationFilePattern = regexp.MustCompile(`^(\d{4})_[a-z0-9_]+\.(up|down)\.sql$`)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user