feat(lab5): durable async attachment pipeline through grade 4

- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
Mikhail Verkovykh
2026-11-15 00:30:36 +03:00
parent 28d68dc30b
commit 9e43faa536
48 changed files with 4077 additions and 333 deletions
@@ -0,0 +1,27 @@
DROP TABLE IF EXISTS attachment_jobs;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_variants_array;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_pipeline_version;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
-- Collapse asynchronous states back to lab4's single in-flight state. Ready
-- and failed rows keep their meaning; queued/processing work is abandoned.
UPDATE attachments
SET status = 'pending', error_code = NULL
WHERE status IN ('queued', 'processing');
ALTER TABLE attachments
DROP COLUMN IF EXISTS variants,
DROP COLUMN IF EXISTS pipeline_version,
DROP COLUMN IF EXISTS staging_key,
DROP COLUMN IF EXISTS ready_at;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status
CHECK (status IN ('pending', 'ready', 'failed')),
ADD CONSTRAINT attachments_error_state CHECK (
(status = 'pending' AND error_code IS NULL) OR
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
);
@@ -0,0 +1,76 @@
-- Lab5 turns attachment processing into a durable asynchronous job.
-- The API stages bytes in S3, then in ONE transaction writes the attachment
-- snapshot and its outbox/job row. A separate publisher moves the intent to
-- RabbitMQ, so a broker outage cannot lose an accepted upload.
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
-- Lab4 "pending" meant "object PUT in progress". Nothing can finish it now,
-- because lab5 acceptors write "queued" plus a job. Preserve the row for the
-- operator instead of silently deleting it.
UPDATE attachments
SET status = 'failed', error_code = 'legacy_interrupted', updated_at = now()
WHERE status = 'pending';
ALTER TABLE attachments
ADD COLUMN IF NOT EXISTS staging_key text,
ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1,
ADD COLUMN IF NOT EXISTS variants jsonb NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS ready_at timestamptz;
UPDATE attachments SET ready_at = updated_at WHERE status = 'ready' AND ready_at IS NULL;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
ADD CONSTRAINT attachments_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
),
ADD CONSTRAINT attachments_variants_array
CHECK (jsonb_typeof(variants) = 'array'),
ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1);
-- One job per logical attachment effect. published_at IS NULL is the outbox
-- marker: a committed intent that has not received a publisher confirm.
CREATE TABLE IF NOT EXISTS attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE,
type text NOT NULL,
pipeline_version integer NOT NULL,
attempt integer NOT NULL DEFAULT 0,
max_attempts integer NOT NULL,
status text NOT NULL,
error_code text,
available_at timestamptz NOT NULL,
published_at timestamptz,
lease_until timestamptz,
processing_started_at timestamptz,
finished_at timestamptz,
replay_count integer NOT NULL DEFAULT 0,
last_replay_at timestamptz,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
CONSTRAINT attachment_jobs_status
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT attachment_jobs_attempts CHECK (attempt >= 0 AND max_attempts >= 1),
CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1),
CONSTRAINT attachment_jobs_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
),
CONSTRAINT attachment_jobs_finished CHECK (
(status IN ('queued', 'processing') AND finished_at IS NULL) OR
(status IN ('ready', 'failed') AND finished_at IS NOT NULL)
)
);
CREATE INDEX IF NOT EXISTS attachment_jobs_outbox_idx
ON attachment_jobs (available_at, created_at)
WHERE published_at IS NULL AND status = 'queued';
CREATE INDEX IF NOT EXISTS attachment_jobs_lease_idx
ON attachment_jobs (lease_until)
WHERE status = 'processing';
CREATE INDEX IF NOT EXISTS attachment_jobs_status_idx
ON attachment_jobs (status, created_at);
@@ -0,0 +1,24 @@
-- Returning to the stricter 0004 shape requires clearing retry reasons first.
UPDATE attachment_jobs
SET error_code = NULL
WHERE status IN ('queued', 'processing');
UPDATE attachments
SET error_code = NULL
WHERE status IN ('queued', 'processing');
ALTER TABLE attachments
DROP CONSTRAINT IF EXISTS attachments_error_state;
ALTER TABLE attachments
ADD CONSTRAINT attachments_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
);
ALTER TABLE attachment_jobs
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
ALTER TABLE attachment_jobs
ADD CONSTRAINT attachment_jobs_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
);
@@ -0,0 +1,23 @@
-- ScheduleRetry records the transient reason on the job while it waits for the
-- next attempt. The original 0004 constraint only allowed error_code on failed
-- rows, so a jittered retry update was rejected and the broker redelivery was
-- then acknowledged as "busy", leaving the job processing until lease expiry.
-- Ready stays clean; failed stays explainable; queued/processing may carry the
-- most recent transient reason.
ALTER TABLE attachments
DROP CONSTRAINT IF EXISTS attachments_error_state;
ALTER TABLE attachments
ADD CONSTRAINT attachments_error_state CHECK (
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL) OR
(status IN ('queued', 'processing'))
);
ALTER TABLE attachment_jobs
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
ALTER TABLE attachment_jobs
ADD CONSTRAINT attachment_jobs_error_state CHECK (
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL) OR
(status IN ('queued', 'processing'))
);
+1 -1
View File
@@ -12,7 +12,7 @@ import (
// latestMigrationVersion is the only value to bump when a migration is added.
// The rest of the test derives filenames and ordering from the directory.
const latestMigrationVersion = 3
const latestMigrationVersion = 5
var migrationFilePattern = regexp.MustCompile(`^(\d{4})_[a-z0-9_]+\.(up|down)\.sql$`)