From efc56b138258994b65aaa631c8e0764384a4869a Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 18 Nov 2026 15:33:32 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A1=D0=BE=D1=85=D1=80=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=82=D0=B5=20=D0=B7=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D1=8F,=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=8B=D1=82=D0=BA=D0=B8=20=D0=B8=20=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D1=81=D0=B8=D0=B8=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B9=20=D0=B2=20PostgreSQL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/pipeline/job.go | 102 +++++++ internal/pipeline/operator.go | 135 +++++++++ internal/pipeline/retry.go | 74 +++++ internal/pipeline/store.go | 142 ++++++++++ internal/pipeline/store_test.go | 334 +++++++++++++++++++++++ migrations/00004_attachment_pipeline.sql | 79 ++++++ 6 files changed, 866 insertions(+) create mode 100644 internal/pipeline/job.go create mode 100644 internal/pipeline/operator.go create mode 100644 internal/pipeline/retry.go create mode 100644 internal/pipeline/store.go create mode 100644 internal/pipeline/store_test.go create mode 100644 migrations/00004_attachment_pipeline.sql diff --git a/internal/pipeline/job.go b/internal/pipeline/job.go new file mode 100644 index 00000000..72618e24 --- /dev/null +++ b/internal/pipeline/job.go @@ -0,0 +1,102 @@ +package pipeline + +import ( + "crypto/sha256" + "errors" + "fmt" + "math/rand/v2" + "regexp" + "time" + + "github.com/google/uuid" +) + +type Job struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +var traceParent = regexp.MustCompile("^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + +func (j Job) Valid() bool { + return j.SchemaVersion == 1 && j.Type == "attachment.process.v1" && + j.JobID != uuid.Nil && j.AttachmentID != uuid.Nil && j.PipelineVersion >= 1 && + j.Attempt >= 1 && !j.CreatedAt.IsZero() && + (j.Traceparent == "" || traceParent.MatchString(j.Traceparent)) && len(j.Tracestate) <= 512 +} + +type Claim struct { + Job + Token uuid.UUID + SourceKey string + Hash []byte + SizeBytes int64 + MIMEType string +} + +type Variant struct { + Name, Key, MIMEType string + Width, Height int + SizeBytes int64 + SHA256 []byte +} + +func VariantKey(id uuid.UUID, version int, name string, hash []byte) string { + return fmt.Sprintf("variants/%s/%d/%s/%x", id, version, name, hash) +} + +func validateVariants(c *Claim, variants []Variant) error { + if len(variants) != 3 { + return errors.New("three variants are required") + } + seen := map[string]bool{} + for _, v := range variants { + mime := "image/jpeg" + switch v.Name { + case "thumbnail", "crop": + case "optimized": + mime = "image/webp" + default: + return errors.New("unknown variant") + } + if seen[v.Name] || v.MIMEType != mime || len(v.SHA256) != sha256.Size || + v.Key != VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, v.SHA256) || + v.Width <= 0 || v.Height <= 0 || v.SizeBytes <= 0 { + return errors.New("invalid variant metadata") + } + seen[v.Name] = true + } + return nil +} + +type RetryPolicy struct { + MaxAttempts int + BaseDelay, MaxDelay time.Duration +} + +func (p RetryPolicy) validate() error { + if p.MaxAttempts < 1 || p.BaseDelay < time.Microsecond || p.MaxDelay < p.BaseDelay { + return errors.New("invalid pipeline retry policy") + } + return nil +} + +// Equal jitter keeps a minimum delay while distributing simultaneous failures. +func (p RetryPolicy) delay(attempt int) time.Duration { + limit := p.BaseDelay + for i := 1; i < attempt && limit < p.MaxDelay; i++ { + if limit > p.MaxDelay/2 { + limit = p.MaxDelay + } else { + limit *= 2 + } + } + return limit/2 + time.Duration(rand.Int64N(int64(limit-limit/2)+1)) +} diff --git a/internal/pipeline/operator.go b/internal/pipeline/operator.go new file mode 100644 index 00000000..be48c696 --- /dev/null +++ b/internal/pipeline/operator.go @@ -0,0 +1,135 @@ +package pipeline + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "messenger/internal/attachment" + "messenger/internal/platform/apperr" +) + +type FailedJob struct { + JobID uuid.UUID `json:"job_id"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + ErrorCode string `json:"error_code"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (s *Store) Failed(ctx context.Context) ([]FailedJob, error) { + rows, err := s.pool.Query(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,error_code,updated_at + FROM attachment_jobs WHERE state='failed' ORDER BY updated_at,job_id`) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, pgx.RowToStructByPos[FailedJob]) +} + +// Lock the attachment before checking the current target, its source or an S3 +// object. This serializes replay and version changes with source collection. +func lockTarget(ctx context.Context, tx pgx.Tx, id uuid.UUID) (int, []byte, error) { + var version int + var hash []byte + err := tx.QueryRow(ctx, `SELECT target_version,file_hash FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&version, &hash) + if errors.Is(err, pgx.ErrNoRows) { + err = apperr.NotFound("attachment not found") + } + return version, hash, err +} + +func availableSource(ctx context.Context, preferred, fallback string, exists func(context.Context, string) (bool, error)) (string, error) { + ok, err := exists(ctx, preferred) + if err != nil { + return "", err + } + if ok { + return preferred, nil + } + if fallback != preferred { + ok, err = exists(ctx, fallback) + if err != nil { + return "", err + } + if ok { + return fallback, nil + } + } + return "", apperr.Conflict("processing source is missing") +} + +func (s *Store) Reprocess(ctx context.Context, id uuid.UUID, version int, exists func(context.Context, string) (bool, error)) error { + if version != 1 && version != 2 { + return apperr.Invalid("unsupported pipeline version") + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + current, hash, err := lockTarget(ctx, tx, id) + if err != nil { + return err + } + if version <= current { + return apperr.Conflict("target version must increase") + } + var source string + if err := tx.QueryRow(ctx, `SELECT source_key FROM attachment_jobs + WHERE attachment_id=$1 AND pipeline_version=$2 FOR UPDATE`, id, current).Scan(&source); err != nil { + return err + } + source, err = availableSource(ctx, attachment.BlobKey(hash), source, exists) + if err != nil { + return err + } + // Keep committed result rows immutable and readable until the switch. + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='superseded',lease_token=NULL,lease_until=NULL,updated_at=clock_timestamp() + WHERE attachment_id=$1 AND pipeline_version=$2 AND state<>'ready'`, id, current); err != nil { + return err + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key) + VALUES ($1,$2,$3)`, id, version, source); err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachments SET target_version=$2 WHERE id=$1`, id, version) + return err + }) +} + +func (s *Store) Replay(ctx context.Context, id uuid.UUID, actor string, exists func(context.Context, string) (bool, error)) error { + actor = strings.TrimSpace(actor) + if actor == "" || len(actor) > 128 { + return apperr.Invalid("replay requires an operator name of up to 128 bytes") + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + current, hash, err := lockTarget(ctx, tx, id) + if err != nil { + return err + } + var jobID uuid.UUID + var source, code string + var attempt int + err = tx.QueryRow(ctx, `SELECT job_id,source_key,error_code,attempt FROM attachment_jobs + WHERE attachment_id=$1 AND pipeline_version=$2 AND state='failed' FOR UPDATE`, id, current). + Scan(&jobID, &source, &code, &attempt) + if errors.Is(err, pgx.ErrNoRows) { + return apperr.Conflict("current job is not failed") + } + if err != nil { + return err + } + source, err = availableSource(ctx, source, attachment.BlobKey(hash), exists) + if err != nil { + return err + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,attempt,actor,previous_error) VALUES ($1,$2,$3,$4)`, + jobID, attempt+1, actor, code); err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempt=attempt+1,budget_attempt=1,source_key=$2, + published_at=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),error_code=NULL WHERE job_id=$1`, jobID, source) + return err + }) +} diff --git a/internal/pipeline/retry.go b/internal/pipeline/retry.go new file mode 100644 index 00000000..0a0fc074 --- /dev/null +++ b/internal/pipeline/retry.go @@ -0,0 +1,74 @@ +package pipeline + +import ( + "context" + "errors" + "regexp" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +var errorCodePattern = regexp.MustCompile("^[a-z][a-z0-9_]{0,63}$") + +func (s *Store) Fail(ctx context.Context, c *Claim, code string, permanent bool, policy RetryPolicy) (bool, error) { + if err := policy.validate(); err != nil { + return false, err + } + if !errorCodePattern.MatchString(code) { + return false, errors.New("invalid failure code") + } + var changed bool + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var budget int + err := tx.QueryRow(ctx, `SELECT budget_attempt FROM attachment_jobs + WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3 AND attempt=$4 + AND state='processing' AND lease_token=$5 AND lease_until>clock_timestamp() FOR UPDATE`, + c.JobID, c.AttachmentID, c.PipelineVersion, c.Attempt, c.Token).Scan(&budget) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + changed, err = finishAttempt(ctx, tx, c.JobID, budget, code, permanent, false, policy) + return err + }) + return changed && err == nil, err +} + +func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error) { + if err := policy.validate(); err != nil { + return false, err + } + var changed bool + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var id uuid.UUID + var budget int + err := tx.QueryRow(ctx, `SELECT job_id,budget_attempt FROM attachment_jobs + WHERE state='processing' AND lease_until<=clock_timestamp() + ORDER BY lease_until,job_id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&id, &budget) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + changed, err = finishAttempt(ctx, tx, id, budget, "lease_expired", false, true, policy) + return err + }) + return changed && err == nil, err +} + +// The caller holds the job row. Only the recovery path may advance an expired +// attempt. A worker whose lease elapsed during its SQL call has lost ownership. +func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, code string, permanent, expired bool, p RetryPolicy) (bool, error) { + retry := !permanent && budget < p.MaxAttempts + tag, err := tx.Exec(ctx, `UPDATE attachment_jobs + SET state=CASE WHEN $3 THEN 'queued' ELSE 'failed' END, + attempt=attempt+$3::boolean::int, budget_attempt=budget_attempt+$3::boolean::int, + available_at=clock_timestamp()+$4::bigint*interval '1 microsecond', + published_at=NULL, lease_token=NULL, lease_until=NULL, error_code=$2, updated_at=clock_timestamp() + WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, code, retry, p.delay(budget).Microseconds(), expired) + return tag.RowsAffected() == 1, err +} diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go new file mode 100644 index 00000000..398239c1 --- /dev/null +++ b/internal/pipeline/store.go @@ -0,0 +1,142 @@ +package pipeline + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" +) + +type Store struct{ pool *pgxpool.Pool } + +func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } + +// The callback waits for a routed publisher confirm with a finite timeout. +// Losing that confirm or the SQL commit leaves the same envelope deliverable. +func (s *Store) PublishOne(ctx context.Context, publish func(context.Context, Job) error) (bool, error) { + var found bool + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + j := Job{SchemaVersion: 1, Type: "attachment.process.v1"} + err := tx.QueryRow(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,created_at + FROM attachment_jobs WHERE state='queued' AND published_at IS NULL AND available_at<=clock_timestamp() + ORDER BY available_at,job_id LIMIT 1 FOR UPDATE SKIP LOCKED`). + Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + found = true + if err := publish(ctx, j); err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET published_at=clock_timestamp() + WHERE job_id=$1 AND attempt=$2 AND state='queued'`, j.JobID, j.Attempt) + return err + }) + return found, err +} + +// Claim checks the envelope against SQL and sources all object metadata from SQL. +// A processing duplicate can be acknowledged: lease recovery is independent +// of broker redelivery and persists a fresh outbox intent after worker death. +func (s *Store) Claim(ctx context.Context, j Job, lease time.Duration) (*Claim, error) { + if !j.Valid() || lease < time.Microsecond { + return nil, errors.New("invalid pipeline job or lease") + } + c := Claim{Job: j, Token: uuid.New()} + err := s.pool.QueryRow(ctx, `UPDATE attachment_jobs j SET state='processing', lease_token=$6, + lease_until=clock_timestamp()+$7::bigint*interval '1 microsecond', updated_at=clock_timestamp() + FROM attachments a JOIN files f ON f.hash=a.file_hash + WHERE j.job_id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.attempt=$4 AND j.created_at=$5 + AND a.id=j.attachment_id AND a.target_version=j.pipeline_version + AND j.state='queued' AND j.available_at<=clock_timestamp() + RETURNING j.source_key,f.hash,f.size_bytes,f.mime_type`, + j.JobID, j.AttachmentID, j.PipelineVersion, j.Attempt, j.CreatedAt, c.Token, lease.Microseconds()). + Scan(&c.SourceKey, &c.Hash, &c.SizeBytes, &c.MIMEType) + if !errors.Is(err, pgx.ErrNoRows) { + if err != nil { + return nil, err + } + return &c, nil + } + var incomplete bool + err = s.pool.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE j.job_id=$1 AND j.state='ready' AND + (coalesce(a.published_version,0)( + SELECT count(*) FROM attachment_variants v WHERE v.attachment_id=a.id AND v.pipeline_version=j.pipeline_version)) + )`, j.JobID).Scan(&incomplete) + if err != nil { + return nil, err + } + if incomplete { + return nil, errors.New("ready job has incomplete durable result") + } + return nil, nil +} + +var errLostClaim = errors.New("claim expired or superseded") + +func (s *Store) Complete(ctx context.Context, c *Claim, variants []Variant) (bool, error) { + if err := validateVariants(c, variants); err != nil { + return false, err + } + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + if err := attachment.LockHash(ctx, tx, c.Hash); err != nil { + return err + } + // Lock the attachment before the job, in the same order as operator commands. + tag, err := tx.Exec(ctx, `SELECT 1 FROM attachments WHERE id=$1 AND target_version=$2 FOR UPDATE`, c.AttachmentID, c.PipelineVersion) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return errLostClaim + } + tag, err = tx.Exec(ctx, `SELECT 1 FROM attachment_jobs WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3 + AND state='processing' AND lease_token=$4 AND lease_until>clock_timestamp() FOR UPDATE`, + c.JobID, c.AttachmentID, c.PipelineVersion, c.Token) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return errLostClaim + } + for _, v := range variants { + _, err := tx.Exec(ctx, `INSERT INTO attachment_variants + (attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`, + c.AttachmentID, c.PipelineVersion, v.Name, v.Key, v.MIMEType, v.Width, v.Height, v.SizeBytes, v.SHA256) + if err != nil { + return err + } + } + if _, err = tx.Exec(ctx, `UPDATE files SET status='ready' WHERE hash=(SELECT file_hash FROM attachments WHERE id=$1)`, c.AttachmentID); err != nil { + return err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET published_version=$2 WHERE id=$1`, c.AttachmentID, c.PipelineVersion); err != nil { + return err + } + // Recheck DB time after every operation that could have waited on a lock. + tag, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='ready',lease_until=NULL,lease_token=NULL, + error_code=NULL,updated_at=clock_timestamp() WHERE job_id=$1 AND lease_until>clock_timestamp()`, c.JobID) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return errLostClaim + } + return nil + }) + if errors.Is(err, errLostClaim) { + return false, nil + } + return err == nil, err +} diff --git a/internal/pipeline/store_test.go b/internal/pipeline/store_test.go new file mode 100644 index 00000000..1c81942d --- /dev/null +++ b/internal/pipeline/store_test.go @@ -0,0 +1,334 @@ +package pipeline + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + + "messenger/internal/testenv" + "messenger/migrations" +) + +func TestPipelineTransitions(t *testing.T) { + ctx := t.Context() + db, err := testenv.StartPostgres(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(db.Close) + s := NewStore(db.Pool) + exists := func(context.Context, string) (bool, error) { return true, nil } + policy := RetryPolicy{MaxAttempts: 2, BaseDelay: time.Millisecond, MaxDelay: time.Second} + exec := func(t *testing.T, query string, args ...any) { + t.Helper() + if _, err := db.Pool.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + newJob := func(t *testing.T) { + t.Helper() + db.Reset(t) + id := uuid.New() + exec(t, `WITH u AS (INSERT INTO users VALUES (gen_random_uuid(),'test','Test User','hash',now()) RETURNING id), + c AS (INSERT INTO chats VALUES (gen_random_uuid(),'chat',now()) RETURNING id), + f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),10,'image/png','pending') RETURNING hash) + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) + SELECT $1,c.id,u.id,'x.png',f.hash,now() FROM u,c,f`, id) + exec(t, `INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, id, "staging/"+id.String()) + } + publish := func(t *testing.T) Job { + t.Helper() + var j Job + ok, err := s.PublishOne(ctx, func(_ context.Context, v Job) error { j = v; return nil }) + if err != nil || !ok || !j.Valid() { + t.Fatalf("publish: %+v, %v, %v", j, ok, err) + } + return j + } + claim := func(t *testing.T, j Job) *Claim { + t.Helper() + c, err := s.Claim(ctx, j, time.Minute) + if err != nil || c == nil { + t.Fatalf("claim: %v %v", c, err) + } + return c + } + variants := func(c *Claim) []Variant { + var vv []Variant + for i, name := range []string{"thumbnail", "crop", "optimized"} { + hash := bytes.Repeat([]byte{byte(c.PipelineVersion*10 + i)}, 32) + mime := "image/jpeg" + if name == "optimized" { + mime = "image/webp" + } + vv = append(vv, Variant{Name: name, Key: fmt.Sprintf("variants/%s/%d/%s/%x", c.AttachmentID, c.PipelineVersion, name, hash), Width: 128, Height: 128, SizeBytes: 4, MIMEType: mime, SHA256: hash}) + } + return vv + } + t.Run("unconfirmed publication is retried and concurrent publishers skip locked work", func(t *testing.T) { + newJob(t) + var seen Job + confirmLost := errors.New("confirm lost") + _, err := s.PublishOne(ctx, func(_ context.Context, j Job) error { + seen = j + ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("double publish") }) + if err != nil || ok { + t.Fatalf("second publisher %v %v", ok, err) + } + return confirmLost + }) + if !errors.Is(err, confirmLost) { + t.Fatal(err) + } + j := publish(t) + if j != seen { + t.Fatalf("redelivery changed envelope: %+v %+v", seen, j) + } + ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("duplicate") }) + if ok || err != nil { + t.Fatalf("confirmed work republished: %v %v", ok, err) + } + }) + t.Run("backoff gates publication and consumption of the next attempt", func(t *testing.T) { + newJob(t) + first := claim(t, publish(t)) + policy := RetryPolicy{MaxAttempts: 3, BaseDelay: 4 * time.Second, MaxDelay: 5 * time.Second} + if ok, err := s.Fail(ctx, first, "storage_timeout", false, policy); !ok || err != nil { + t.Fatalf("failed %v %v", ok, err) + } + var seconds float64 + if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil { + t.Fatal(err) + } + if seconds < 1.99 || seconds > 4.01 { + t.Fatalf("first retry delay %.3f seconds", seconds) + } + next := first.Job + next.Attempt++ + if c, err := s.Claim(ctx, next, time.Minute); err != nil || c != nil { + t.Fatalf("premature claim %v %v", c, err) + } + if ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("premature publish") }); err != nil || ok { + t.Fatalf("premature publish %v %v", ok, err) + } + exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) + second := claim(t, publish(t)) + if ok, err := s.Fail(ctx, second, "storage_timeout", false, policy); !ok || err != nil { + t.Fatalf("second failure %v %v", ok, err) + } + if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil { + t.Fatal(err) + } + if seconds < 2.49 || seconds > 5.01 { + t.Fatalf("capped second retry delay %.3f seconds", seconds) + } + }) + t.Run("parallel duplicates and version change before commit", func(t *testing.T) { + newJob(t) + j := publish(t) + var wg sync.WaitGroup + claims := make(chan *Claim, 8) + for range 8 { + wg.Go(func() { + c, err := s.Claim(ctx, j, time.Minute) + if err != nil { + t.Error(err) + } + if c != nil { + claims <- c + } + }) + } + wg.Wait() + close(claims) + if len(claims) != 1 { + t.Fatalf("%d workers claimed one job", len(claims)) + } + old := <-claims + if old.SourceKey != "staging/"+j.AttachmentID.String() || old.SizeBytes != 10 || !bytes.Equal(old.Hash, bytes.Repeat([]byte{0xab}, 32)) { + t.Fatalf("source metadata: %+v", old) + } + stageOnly := func(_ context.Context, key string) (bool, error) { return key == old.SourceKey, nil } + // A different attachment can have published the same digest. That row's + // status cannot prove the object is still there when the target changes. + exec(t, `UPDATE files SET status='ready'`) + if err := s.Reprocess(ctx, j.AttachmentID, 2, stageOnly); err != nil { + t.Fatal(err) + } + current := claim(t, publish(t)) + if current.SourceKey != old.SourceKey { + t.Fatalf("available staging ignored in favor of missing CAS: %s", current.SourceKey) + } + results := variants(current) + if _, err := s.Complete(ctx, current, results[:2]); err == nil { + t.Fatal("published incomplete result") + } + if ok, err := s.Complete(ctx, current, results); err != nil || !ok { + t.Fatalf("complete %v %v", ok, err) + } + if ok, err := s.Complete(ctx, old, variants(old)); err != nil || ok { + t.Fatalf("stale commit %v %v", ok, err) + } + if ok, err := s.Fail(ctx, old, "source_missing", true, policy); err != nil || ok { + t.Fatalf("stale failure %v %v", ok, err) + } + for _, dup := range []Job{current.Job, j, current.Job} { + if c, err := s.Claim(ctx, dup, time.Minute); err != nil || c != nil { + t.Fatalf("duplicate %v %v", c, err) + } + } + var version, count int + var digest []byte + if err := db.Pool.QueryRow(ctx, `SELECT a.published_version, + (SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id), + (SELECT sha256 FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2 AND variant='optimized') + FROM attachments a WHERE a.id=$1`, j.AttachmentID).Scan(&version, &count, &digest); err != nil { + t.Fatal(err) + } + if version != 2 || count != 3 || !bytes.Equal(digest, results[2].SHA256) { + t.Fatalf("published v%d count %d hash %x", version, count, digest) + } + exec(t, `DELETE FROM attachment_variants WHERE variant='crop'`) + if _, err := s.Claim(ctx, current.Job, time.Minute); err == nil { + t.Fatal("ready duplicate acknowledged without complete result") + } + }) + t.Run("dead lease and retry exhaustion then audited replay", func(t *testing.T) { + newJob(t) + first := claim(t, publish(t)) + exec(t, `UPDATE attachment_jobs SET lease_until=clock_timestamp()-interval '1 second'`) + if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok { + t.Fatalf("expired commit %v %v", ok, err) + } + if ok, err := s.Fail(ctx, first, "backend_unavailable", false, policy); err != nil || ok { + t.Fatalf("expired fail %v %v", ok, err) + } + if ok, err := s.RecoverOne(ctx, policy); err != nil || !ok { + t.Fatalf("recover %v %v", ok, err) + } + exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) + second := claim(t, publish(t)) + if first.JobID != second.JobID || first.Attempt != 1 || second.Attempt != 2 || first.Token == second.Token { + t.Fatalf("attempt identity %+v %+v", first, second) + } + if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok { + t.Fatalf("lost lease commit %v %v", ok, err) + } + if ok, err := s.Fail(ctx, second, "backend_unavailable", false, policy); err != nil || !ok { + t.Fatalf("second error %v %v", ok, err) + } + failed, err := s.Failed(ctx) + if err != nil || len(failed) != 1 { + t.Fatalf("failed %v %v", failed, err) + } + absent := func(context.Context, string) (bool, error) { return false, nil } + if err := s.Replay(ctx, first.AttachmentID, "test", absent); err == nil { + t.Fatal("replayed without source") + } + hasStage := func(_ context.Context, k string) (bool, error) { return k == first.SourceKey, nil } + if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err != nil { + t.Fatal(err) + } + if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err == nil { + t.Fatal("replayed active job") + } + if c, err := s.Claim(ctx, first.Job, time.Minute); c != nil || err != nil { + t.Fatalf("old delivery %v %v", c, err) + } + replay := claim(t, publish(t)) + if replay.Attempt != 3 || replay.JobID != first.JobID { + t.Fatalf("replay identity %+v", replay) + } + if ok, err := s.Fail(ctx, replay, "backend_unavailable", false, policy); !ok || err != nil { + t.Fatalf("fresh retry budget %v %v", ok, err) + } + exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) + last := claim(t, publish(t)) + if ok, err := s.Fail(ctx, last, "source_missing", true, policy); !ok || err != nil { + t.Fatalf("permanent fail %v %v", ok, err) + } + var n int + if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachment_replays WHERE actor='operator' AND previous_error='backend_unavailable'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("replay audit %d %v", n, err) + } + }) + t.Run("the previous version remains readable until the entire new result commits", func(t *testing.T) { + newJob(t) + old := claim(t, publish(t)) + if ok, err := s.Complete(ctx, old, variants(old)); err != nil || !ok { + t.Fatalf("first publication %v %v", ok, err) + } + if err := s.Reprocess(ctx, old.AttachmentID, 2, exists); err != nil { + t.Fatal(err) + } + next := claim(t, publish(t)) + if next.SourceKey != fmt.Sprintf("blobs/%x", old.Hash) { + t.Fatalf("version two source %s", next.SourceKey) + } + // Force an actual SQL failure on the second INSERT, after the first one succeeded. + exec(t, `ALTER TABLE attachment_variants ADD CONSTRAINT experiment_failure CHECK (pipeline_version<>2 OR variant<>'crop')`) + if _, err := s.Complete(ctx, next, variants(next)); err == nil { + t.Fatal("expected SQL failure on second variant") + } + exec(t, `ALTER TABLE attachment_variants DROP CONSTRAINT experiment_failure`) + var version, resultCount int + var status string + if err := db.Pool.QueryRow(ctx, `SELECT a.published_version,s.status, + (SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2) + FROM attachment_states s JOIN attachments a USING(id)`).Scan(&version, &status, &resultCount); err != nil { + t.Fatal(err) + } + if version != 1 || status != "ready" || resultCount != 0 { + t.Fatalf("partial publication v%d status=%s rows=%d", version, status, resultCount) + } + if ok, err := s.Fail(ctx, next, "invalid_image", true, policy); err != nil || !ok { + t.Fatalf("permanent error %v %v", ok, err) + } + failed, err := s.Failed(ctx) + if err != nil || len(failed) != 1 || failed[0].Attempt != 1 || failed[0].ErrorCode != "invalid_image" { + t.Fatalf("permanent error not terminal %v %v", failed, err) + } + if err := db.Pool.QueryRow(ctx, `SELECT status FROM attachment_states`).Scan(&status); err != nil || status != "ready" { + t.Fatalf("failed reprocess hid original %s %v", status, err) + } + }) + t.Run("migration preserves Lab4 message links and requeues known originals", func(t *testing.T) { + newJob(t) + provider, err := goose.NewProvider(goose.DialectPostgres, stdlib.OpenDBFromPool(db.Pool), migrations.FS) + if err != nil { + t.Fatal(err) + } + defer provider.Close() + if _, err := provider.DownTo(ctx, 3); err != nil { + t.Fatal(err) + } + exec(t, `UPDATE files SET status='ready'`) + exec(t, `WITH f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('cd',32),'hex'),10,'image/jpeg','failed') RETURNING hash) + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),chat_id,owner_id,'missing.jpg',f.hash,now() FROM attachments,f`) + exec(t, `WITH m AS (INSERT INTO messages(chat_id,id,sender_id,text,created_at) SELECT chat_id,gen_random_uuid(),owner_id,'old',now() FROM attachments LIMIT 1 RETURNING chat_id,id) + INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) SELECT m.chat_id,m.id,a.id,0 FROM m,attachments a WHERE a.filename='x.png'`) + if _, err := provider.Up(ctx); err != nil { + t.Fatal(err) + } + job := claim(t, publish(t)) + if job.SourceKey != fmt.Sprintf("blobs/%x", bytes.Repeat([]byte{0xab}, 32)) { + t.Fatalf("migrated source %s", job.SourceKey) + } + var linked uuid.UUID + if err := db.Pool.QueryRow(ctx, `SELECT attachment_id FROM message_attachments`).Scan(&linked); err != nil || linked != job.AttachmentID { + t.Fatalf("changed link %s %v", linked, err) + } + failed, err := s.Failed(ctx) + if err != nil || len(failed) != 1 || failed[0].ErrorCode != "source_missing" { + t.Fatalf("migration failed registry %v %v", failed, err) + } + }) +} diff --git a/migrations/00004_attachment_pipeline.sql b/migrations/00004_attachment_pipeline.sql new file mode 100644 index 00000000..eb3f0684 --- /dev/null +++ b/migrations/00004_attachment_pipeline.sql @@ -0,0 +1,79 @@ +-- +goose Up +ALTER TABLE attachments + ADD COLUMN target_version integer NOT NULL DEFAULT 1 CHECK (target_version >= 1), + ADD COLUMN published_version integer CHECK (published_version BETWEEN 1 AND target_version); + +-- A queued row with no publish timestamp is an outbox entry for this attempt. +-- attempt never decreases. The separate budget starts again only on operator replay. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')), + source_key text NOT NULL, + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + budget_attempt integer NOT NULL DEFAULT 1 CHECK (budget_attempt BETWEEN 1 AND attempt), + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + lease_token uuid, + lease_until timestamptz, + error_code text CHECK (error_code ~ '^[a-z][a-z0-9_]{0,63}$'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version), + CHECK ((state = 'processing') = (lease_token IS NOT NULL)), + CHECK ((state = 'processing') = (lease_until IS NOT NULL)), + CHECK (state <> 'failed' OR error_code IS NOT NULL) +); +CREATE INDEX attachment_jobs_outbox_idx ON attachment_jobs (available_at, job_id) + WHERE state='queued' AND published_at IS NULL; +CREATE INDEX attachment_jobs_recover_idx ON attachment_jobs (lease_until, job_id) WHERE state='processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at, job_id) WHERE state='failed'; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + variant text NOT NULL CHECK (variant IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL, + mime_type text NOT NULL CHECK (mime_type = CASE WHEN variant='optimized' THEN 'image/webp' ELSE 'image/jpeg' END), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + sha256 bytea NOT NULL CHECK (octet_length(sha256)=32), + PRIMARY KEY (attachment_id,pipeline_version,variant), + FOREIGN KEY (attachment_id,pipeline_version) REFERENCES attachment_jobs(attachment_id,pipeline_version) +); + +CREATE TABLE attachment_replays ( + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id), + attempt integer NOT NULL, + actor text NOT NULL CHECK (length(actor) BETWEEN 1 AND 128), + previous_error text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, attempt) +); + +-- Published Lab 4 objects are valid inputs. A recorded missing original +-- stays diagnosable even before a worker is started. +INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code) +SELECT a.id,'blobs/'||encode(f.hash,'hex'), + CASE WHEN f.status='failed' THEN 'failed' ELSE 'queued' END, + CASE WHEN f.status='failed' THEN 'source_missing' END +FROM attachments a JOIN files f ON f.hash=a.file_hash; + +CREATE VIEW attachment_states AS +SELECT a.id, + CASE WHEN a.published_version IS NULL THEN j.state + WHEN f.status='failed' THEN 'failed' ELSE 'ready' END AS status, + CASE WHEN a.published_version IS NOT NULL AND f.status='failed' THEN 'source_missing' + WHEN a.published_version IS NULL AND j.state='failed' THEN j.error_code END AS error_code, + j.updated_at +FROM attachments a JOIN files f ON f.hash=a.file_hash +JOIN attachment_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.target_version; + +-- +goose Down +DROP VIEW attachment_states; +DROP TABLE attachment_replays; +DROP TABLE attachment_variants; +DROP TABLE attachment_jobs; +ALTER TABLE attachments DROP COLUMN published_version, DROP COLUMN target_version;