Сохраните задания, попытки и версии вложений в PostgreSQL

This commit is contained in:
2026-11-18 15:33:32 +03:00
parent 911d97a77a
commit efc56b1382
6 changed files with 866 additions and 0 deletions
+102
View File
@@ -0,0 +1,102 @@
package pipeline
import (
"crypto/sha256"
"errors"
"fmt"
"math/rand/v2"
"regexp"
"time"
"github.com/google/uuid"
)
type Job struct {
SchemaVersion int `json:"schema_version"`
JobID uuid.UUID `json:"job_id"`
Type string `json:"type"`
AttachmentID uuid.UUID `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
}
var traceParent = regexp.MustCompile("^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
func (j Job) Valid() bool {
return j.SchemaVersion == 1 && j.Type == "attachment.process.v1" &&
j.JobID != uuid.Nil && j.AttachmentID != uuid.Nil && j.PipelineVersion >= 1 &&
j.Attempt >= 1 && !j.CreatedAt.IsZero() &&
(j.Traceparent == "" || traceParent.MatchString(j.Traceparent)) && len(j.Tracestate) <= 512
}
type Claim struct {
Job
Token uuid.UUID
SourceKey string
Hash []byte
SizeBytes int64
MIMEType string
}
type Variant struct {
Name, Key, MIMEType string
Width, Height int
SizeBytes int64
SHA256 []byte
}
func VariantKey(id uuid.UUID, version int, name string, hash []byte) string {
return fmt.Sprintf("variants/%s/%d/%s/%x", id, version, name, hash)
}
func validateVariants(c *Claim, variants []Variant) error {
if len(variants) != 3 {
return errors.New("three variants are required")
}
seen := map[string]bool{}
for _, v := range variants {
mime := "image/jpeg"
switch v.Name {
case "thumbnail", "crop":
case "optimized":
mime = "image/webp"
default:
return errors.New("unknown variant")
}
if seen[v.Name] || v.MIMEType != mime || len(v.SHA256) != sha256.Size ||
v.Key != VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, v.SHA256) ||
v.Width <= 0 || v.Height <= 0 || v.SizeBytes <= 0 {
return errors.New("invalid variant metadata")
}
seen[v.Name] = true
}
return nil
}
type RetryPolicy struct {
MaxAttempts int
BaseDelay, MaxDelay time.Duration
}
func (p RetryPolicy) validate() error {
if p.MaxAttempts < 1 || p.BaseDelay < time.Microsecond || p.MaxDelay < p.BaseDelay {
return errors.New("invalid pipeline retry policy")
}
return nil
}
// Equal jitter keeps a minimum delay while distributing simultaneous failures.
func (p RetryPolicy) delay(attempt int) time.Duration {
limit := p.BaseDelay
for i := 1; i < attempt && limit < p.MaxDelay; i++ {
if limit > p.MaxDelay/2 {
limit = p.MaxDelay
} else {
limit *= 2
}
}
return limit/2 + time.Duration(rand.Int64N(int64(limit-limit/2)+1))
}
+135
View File
@@ -0,0 +1,135 @@
package pipeline
import (
"context"
"errors"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"messenger/internal/attachment"
"messenger/internal/platform/apperr"
)
type FailedJob struct {
JobID uuid.UUID `json:"job_id"`
AttachmentID uuid.UUID `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
ErrorCode string `json:"error_code"`
UpdatedAt time.Time `json:"updated_at"`
}
func (s *Store) Failed(ctx context.Context) ([]FailedJob, error) {
rows, err := s.pool.Query(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,error_code,updated_at
FROM attachment_jobs WHERE state='failed' ORDER BY updated_at,job_id`)
if err != nil {
return nil, err
}
return pgx.CollectRows(rows, pgx.RowToStructByPos[FailedJob])
}
// Lock the attachment before checking the current target, its source or an S3
// object. This serializes replay and version changes with source collection.
func lockTarget(ctx context.Context, tx pgx.Tx, id uuid.UUID) (int, []byte, error) {
var version int
var hash []byte
err := tx.QueryRow(ctx, `SELECT target_version,file_hash FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&version, &hash)
if errors.Is(err, pgx.ErrNoRows) {
err = apperr.NotFound("attachment not found")
}
return version, hash, err
}
func availableSource(ctx context.Context, preferred, fallback string, exists func(context.Context, string) (bool, error)) (string, error) {
ok, err := exists(ctx, preferred)
if err != nil {
return "", err
}
if ok {
return preferred, nil
}
if fallback != preferred {
ok, err = exists(ctx, fallback)
if err != nil {
return "", err
}
if ok {
return fallback, nil
}
}
return "", apperr.Conflict("processing source is missing")
}
func (s *Store) Reprocess(ctx context.Context, id uuid.UUID, version int, exists func(context.Context, string) (bool, error)) error {
if version != 1 && version != 2 {
return apperr.Invalid("unsupported pipeline version")
}
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
current, hash, err := lockTarget(ctx, tx, id)
if err != nil {
return err
}
if version <= current {
return apperr.Conflict("target version must increase")
}
var source string
if err := tx.QueryRow(ctx, `SELECT source_key FROM attachment_jobs
WHERE attachment_id=$1 AND pipeline_version=$2 FOR UPDATE`, id, current).Scan(&source); err != nil {
return err
}
source, err = availableSource(ctx, attachment.BlobKey(hash), source, exists)
if err != nil {
return err
}
// Keep committed result rows immutable and readable until the switch.
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='superseded',lease_token=NULL,lease_until=NULL,updated_at=clock_timestamp()
WHERE attachment_id=$1 AND pipeline_version=$2 AND state<>'ready'`, id, current); err != nil {
return err
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key)
VALUES ($1,$2,$3)`, id, version, source); err != nil {
return err
}
_, err = tx.Exec(ctx, `UPDATE attachments SET target_version=$2 WHERE id=$1`, id, version)
return err
})
}
func (s *Store) Replay(ctx context.Context, id uuid.UUID, actor string, exists func(context.Context, string) (bool, error)) error {
actor = strings.TrimSpace(actor)
if actor == "" || len(actor) > 128 {
return apperr.Invalid("replay requires an operator name of up to 128 bytes")
}
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
current, hash, err := lockTarget(ctx, tx, id)
if err != nil {
return err
}
var jobID uuid.UUID
var source, code string
var attempt int
err = tx.QueryRow(ctx, `SELECT job_id,source_key,error_code,attempt FROM attachment_jobs
WHERE attachment_id=$1 AND pipeline_version=$2 AND state='failed' FOR UPDATE`, id, current).
Scan(&jobID, &source, &code, &attempt)
if errors.Is(err, pgx.ErrNoRows) {
return apperr.Conflict("current job is not failed")
}
if err != nil {
return err
}
source, err = availableSource(ctx, source, attachment.BlobKey(hash), exists)
if err != nil {
return err
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,attempt,actor,previous_error) VALUES ($1,$2,$3,$4)`,
jobID, attempt+1, actor, code); err != nil {
return err
}
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempt=attempt+1,budget_attempt=1,source_key=$2,
published_at=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),error_code=NULL WHERE job_id=$1`, jobID, source)
return err
})
}
+74
View File
@@ -0,0 +1,74 @@
package pipeline
import (
"context"
"errors"
"regexp"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
var errorCodePattern = regexp.MustCompile("^[a-z][a-z0-9_]{0,63}$")
func (s *Store) Fail(ctx context.Context, c *Claim, code string, permanent bool, policy RetryPolicy) (bool, error) {
if err := policy.validate(); err != nil {
return false, err
}
if !errorCodePattern.MatchString(code) {
return false, errors.New("invalid failure code")
}
var changed bool
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
var budget int
err := tx.QueryRow(ctx, `SELECT budget_attempt FROM attachment_jobs
WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3 AND attempt=$4
AND state='processing' AND lease_token=$5 AND lease_until>clock_timestamp() FOR UPDATE`,
c.JobID, c.AttachmentID, c.PipelineVersion, c.Attempt, c.Token).Scan(&budget)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
changed, err = finishAttempt(ctx, tx, c.JobID, budget, code, permanent, false, policy)
return err
})
return changed && err == nil, err
}
func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error) {
if err := policy.validate(); err != nil {
return false, err
}
var changed bool
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
var id uuid.UUID
var budget int
err := tx.QueryRow(ctx, `SELECT job_id,budget_attempt FROM attachment_jobs
WHERE state='processing' AND lease_until<=clock_timestamp()
ORDER BY lease_until,job_id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&id, &budget)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
changed, err = finishAttempt(ctx, tx, id, budget, "lease_expired", false, true, policy)
return err
})
return changed && err == nil, err
}
// The caller holds the job row. Only the recovery path may advance an expired
// attempt. A worker whose lease elapsed during its SQL call has lost ownership.
func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, code string, permanent, expired bool, p RetryPolicy) (bool, error) {
retry := !permanent && budget < p.MaxAttempts
tag, err := tx.Exec(ctx, `UPDATE attachment_jobs
SET state=CASE WHEN $3 THEN 'queued' ELSE 'failed' END,
attempt=attempt+$3::boolean::int, budget_attempt=budget_attempt+$3::boolean::int,
available_at=clock_timestamp()+$4::bigint*interval '1 microsecond',
published_at=NULL, lease_token=NULL, lease_until=NULL, error_code=$2, updated_at=clock_timestamp()
WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, code, retry, p.delay(budget).Microseconds(), expired)
return tag.RowsAffected() == 1, err
}
+142
View File
@@ -0,0 +1,142 @@
package pipeline
import (
"context"
"errors"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/attachment"
)
type Store struct{ pool *pgxpool.Pool }
func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
// The callback waits for a routed publisher confirm with a finite timeout.
// Losing that confirm or the SQL commit leaves the same envelope deliverable.
func (s *Store) PublishOne(ctx context.Context, publish func(context.Context, Job) error) (bool, error) {
var found bool
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
j := Job{SchemaVersion: 1, Type: "attachment.process.v1"}
err := tx.QueryRow(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,created_at
FROM attachment_jobs WHERE state='queued' AND published_at IS NULL AND available_at<=clock_timestamp()
ORDER BY available_at,job_id LIMIT 1 FOR UPDATE SKIP LOCKED`).
Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
found = true
if err := publish(ctx, j); err != nil {
return err
}
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET published_at=clock_timestamp()
WHERE job_id=$1 AND attempt=$2 AND state='queued'`, j.JobID, j.Attempt)
return err
})
return found, err
}
// Claim checks the envelope against SQL and sources all object metadata from SQL.
// A processing duplicate can be acknowledged: lease recovery is independent
// of broker redelivery and persists a fresh outbox intent after worker death.
func (s *Store) Claim(ctx context.Context, j Job, lease time.Duration) (*Claim, error) {
if !j.Valid() || lease < time.Microsecond {
return nil, errors.New("invalid pipeline job or lease")
}
c := Claim{Job: j, Token: uuid.New()}
err := s.pool.QueryRow(ctx, `UPDATE attachment_jobs j SET state='processing', lease_token=$6,
lease_until=clock_timestamp()+$7::bigint*interval '1 microsecond', updated_at=clock_timestamp()
FROM attachments a JOIN files f ON f.hash=a.file_hash
WHERE j.job_id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.attempt=$4 AND j.created_at=$5
AND a.id=j.attachment_id AND a.target_version=j.pipeline_version
AND j.state='queued' AND j.available_at<=clock_timestamp()
RETURNING j.source_key,f.hash,f.size_bytes,f.mime_type`,
j.JobID, j.AttachmentID, j.PipelineVersion, j.Attempt, j.CreatedAt, c.Token, lease.Microseconds()).
Scan(&c.SourceKey, &c.Hash, &c.SizeBytes, &c.MIMEType)
if !errors.Is(err, pgx.ErrNoRows) {
if err != nil {
return nil, err
}
return &c, nil
}
var incomplete bool
err = s.pool.QueryRow(ctx, `SELECT EXISTS (
SELECT 1 FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE j.job_id=$1 AND j.state='ready' AND
(coalesce(a.published_version,0)<j.pipeline_version OR 3<>(
SELECT count(*) FROM attachment_variants v WHERE v.attachment_id=a.id AND v.pipeline_version=j.pipeline_version))
)`, j.JobID).Scan(&incomplete)
if err != nil {
return nil, err
}
if incomplete {
return nil, errors.New("ready job has incomplete durable result")
}
return nil, nil
}
var errLostClaim = errors.New("claim expired or superseded")
func (s *Store) Complete(ctx context.Context, c *Claim, variants []Variant) (bool, error) {
if err := validateVariants(c, variants); err != nil {
return false, err
}
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if err := attachment.LockHash(ctx, tx, c.Hash); err != nil {
return err
}
// Lock the attachment before the job, in the same order as operator commands.
tag, err := tx.Exec(ctx, `SELECT 1 FROM attachments WHERE id=$1 AND target_version=$2 FOR UPDATE`, c.AttachmentID, c.PipelineVersion)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return errLostClaim
}
tag, err = tx.Exec(ctx, `SELECT 1 FROM attachment_jobs WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3
AND state='processing' AND lease_token=$4 AND lease_until>clock_timestamp() FOR UPDATE`,
c.JobID, c.AttachmentID, c.PipelineVersion, c.Token)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return errLostClaim
}
for _, v := range variants {
_, err := tx.Exec(ctx, `INSERT INTO attachment_variants
(attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
c.AttachmentID, c.PipelineVersion, v.Name, v.Key, v.MIMEType, v.Width, v.Height, v.SizeBytes, v.SHA256)
if err != nil {
return err
}
}
if _, err = tx.Exec(ctx, `UPDATE files SET status='ready' WHERE hash=(SELECT file_hash FROM attachments WHERE id=$1)`, c.AttachmentID); err != nil {
return err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET published_version=$2 WHERE id=$1`, c.AttachmentID, c.PipelineVersion); err != nil {
return err
}
// Recheck DB time after every operation that could have waited on a lock.
tag, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='ready',lease_until=NULL,lease_token=NULL,
error_code=NULL,updated_at=clock_timestamp() WHERE job_id=$1 AND lease_until>clock_timestamp()`, c.JobID)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return errLostClaim
}
return nil
})
if errors.Is(err, errLostClaim) {
return false, nil
}
return err == nil, err
}
+334
View File
@@ -0,0 +1,334 @@
package pipeline
import (
"bytes"
"context"
"errors"
"fmt"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/stdlib"
"github.com/pressly/goose/v3"
"messenger/internal/testenv"
"messenger/migrations"
)
func TestPipelineTransitions(t *testing.T) {
ctx := t.Context()
db, err := testenv.StartPostgres(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(db.Close)
s := NewStore(db.Pool)
exists := func(context.Context, string) (bool, error) { return true, nil }
policy := RetryPolicy{MaxAttempts: 2, BaseDelay: time.Millisecond, MaxDelay: time.Second}
exec := func(t *testing.T, query string, args ...any) {
t.Helper()
if _, err := db.Pool.Exec(ctx, query, args...); err != nil {
t.Fatal(err)
}
}
newJob := func(t *testing.T) {
t.Helper()
db.Reset(t)
id := uuid.New()
exec(t, `WITH u AS (INSERT INTO users VALUES (gen_random_uuid(),'test','Test User','hash',now()) RETURNING id),
c AS (INSERT INTO chats VALUES (gen_random_uuid(),'chat',now()) RETURNING id),
f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),10,'image/png','pending') RETURNING hash)
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at)
SELECT $1,c.id,u.id,'x.png',f.hash,now() FROM u,c,f`, id)
exec(t, `INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, id, "staging/"+id.String())
}
publish := func(t *testing.T) Job {
t.Helper()
var j Job
ok, err := s.PublishOne(ctx, func(_ context.Context, v Job) error { j = v; return nil })
if err != nil || !ok || !j.Valid() {
t.Fatalf("publish: %+v, %v, %v", j, ok, err)
}
return j
}
claim := func(t *testing.T, j Job) *Claim {
t.Helper()
c, err := s.Claim(ctx, j, time.Minute)
if err != nil || c == nil {
t.Fatalf("claim: %v %v", c, err)
}
return c
}
variants := func(c *Claim) []Variant {
var vv []Variant
for i, name := range []string{"thumbnail", "crop", "optimized"} {
hash := bytes.Repeat([]byte{byte(c.PipelineVersion*10 + i)}, 32)
mime := "image/jpeg"
if name == "optimized" {
mime = "image/webp"
}
vv = append(vv, Variant{Name: name, Key: fmt.Sprintf("variants/%s/%d/%s/%x", c.AttachmentID, c.PipelineVersion, name, hash), Width: 128, Height: 128, SizeBytes: 4, MIMEType: mime, SHA256: hash})
}
return vv
}
t.Run("unconfirmed publication is retried and concurrent publishers skip locked work", func(t *testing.T) {
newJob(t)
var seen Job
confirmLost := errors.New("confirm lost")
_, err := s.PublishOne(ctx, func(_ context.Context, j Job) error {
seen = j
ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("double publish") })
if err != nil || ok {
t.Fatalf("second publisher %v %v", ok, err)
}
return confirmLost
})
if !errors.Is(err, confirmLost) {
t.Fatal(err)
}
j := publish(t)
if j != seen {
t.Fatalf("redelivery changed envelope: %+v %+v", seen, j)
}
ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("duplicate") })
if ok || err != nil {
t.Fatalf("confirmed work republished: %v %v", ok, err)
}
})
t.Run("backoff gates publication and consumption of the next attempt", func(t *testing.T) {
newJob(t)
first := claim(t, publish(t))
policy := RetryPolicy{MaxAttempts: 3, BaseDelay: 4 * time.Second, MaxDelay: 5 * time.Second}
if ok, err := s.Fail(ctx, first, "storage_timeout", false, policy); !ok || err != nil {
t.Fatalf("failed %v %v", ok, err)
}
var seconds float64
if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil {
t.Fatal(err)
}
if seconds < 1.99 || seconds > 4.01 {
t.Fatalf("first retry delay %.3f seconds", seconds)
}
next := first.Job
next.Attempt++
if c, err := s.Claim(ctx, next, time.Minute); err != nil || c != nil {
t.Fatalf("premature claim %v %v", c, err)
}
if ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("premature publish") }); err != nil || ok {
t.Fatalf("premature publish %v %v", ok, err)
}
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
second := claim(t, publish(t))
if ok, err := s.Fail(ctx, second, "storage_timeout", false, policy); !ok || err != nil {
t.Fatalf("second failure %v %v", ok, err)
}
if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil {
t.Fatal(err)
}
if seconds < 2.49 || seconds > 5.01 {
t.Fatalf("capped second retry delay %.3f seconds", seconds)
}
})
t.Run("parallel duplicates and version change before commit", func(t *testing.T) {
newJob(t)
j := publish(t)
var wg sync.WaitGroup
claims := make(chan *Claim, 8)
for range 8 {
wg.Go(func() {
c, err := s.Claim(ctx, j, time.Minute)
if err != nil {
t.Error(err)
}
if c != nil {
claims <- c
}
})
}
wg.Wait()
close(claims)
if len(claims) != 1 {
t.Fatalf("%d workers claimed one job", len(claims))
}
old := <-claims
if old.SourceKey != "staging/"+j.AttachmentID.String() || old.SizeBytes != 10 || !bytes.Equal(old.Hash, bytes.Repeat([]byte{0xab}, 32)) {
t.Fatalf("source metadata: %+v", old)
}
stageOnly := func(_ context.Context, key string) (bool, error) { return key == old.SourceKey, nil }
// A different attachment can have published the same digest. That row's
// status cannot prove the object is still there when the target changes.
exec(t, `UPDATE files SET status='ready'`)
if err := s.Reprocess(ctx, j.AttachmentID, 2, stageOnly); err != nil {
t.Fatal(err)
}
current := claim(t, publish(t))
if current.SourceKey != old.SourceKey {
t.Fatalf("available staging ignored in favor of missing CAS: %s", current.SourceKey)
}
results := variants(current)
if _, err := s.Complete(ctx, current, results[:2]); err == nil {
t.Fatal("published incomplete result")
}
if ok, err := s.Complete(ctx, current, results); err != nil || !ok {
t.Fatalf("complete %v %v", ok, err)
}
if ok, err := s.Complete(ctx, old, variants(old)); err != nil || ok {
t.Fatalf("stale commit %v %v", ok, err)
}
if ok, err := s.Fail(ctx, old, "source_missing", true, policy); err != nil || ok {
t.Fatalf("stale failure %v %v", ok, err)
}
for _, dup := range []Job{current.Job, j, current.Job} {
if c, err := s.Claim(ctx, dup, time.Minute); err != nil || c != nil {
t.Fatalf("duplicate %v %v", c, err)
}
}
var version, count int
var digest []byte
if err := db.Pool.QueryRow(ctx, `SELECT a.published_version,
(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id),
(SELECT sha256 FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2 AND variant='optimized')
FROM attachments a WHERE a.id=$1`, j.AttachmentID).Scan(&version, &count, &digest); err != nil {
t.Fatal(err)
}
if version != 2 || count != 3 || !bytes.Equal(digest, results[2].SHA256) {
t.Fatalf("published v%d count %d hash %x", version, count, digest)
}
exec(t, `DELETE FROM attachment_variants WHERE variant='crop'`)
if _, err := s.Claim(ctx, current.Job, time.Minute); err == nil {
t.Fatal("ready duplicate acknowledged without complete result")
}
})
t.Run("dead lease and retry exhaustion then audited replay", func(t *testing.T) {
newJob(t)
first := claim(t, publish(t))
exec(t, `UPDATE attachment_jobs SET lease_until=clock_timestamp()-interval '1 second'`)
if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok {
t.Fatalf("expired commit %v %v", ok, err)
}
if ok, err := s.Fail(ctx, first, "backend_unavailable", false, policy); err != nil || ok {
t.Fatalf("expired fail %v %v", ok, err)
}
if ok, err := s.RecoverOne(ctx, policy); err != nil || !ok {
t.Fatalf("recover %v %v", ok, err)
}
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
second := claim(t, publish(t))
if first.JobID != second.JobID || first.Attempt != 1 || second.Attempt != 2 || first.Token == second.Token {
t.Fatalf("attempt identity %+v %+v", first, second)
}
if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok {
t.Fatalf("lost lease commit %v %v", ok, err)
}
if ok, err := s.Fail(ctx, second, "backend_unavailable", false, policy); err != nil || !ok {
t.Fatalf("second error %v %v", ok, err)
}
failed, err := s.Failed(ctx)
if err != nil || len(failed) != 1 {
t.Fatalf("failed %v %v", failed, err)
}
absent := func(context.Context, string) (bool, error) { return false, nil }
if err := s.Replay(ctx, first.AttachmentID, "test", absent); err == nil {
t.Fatal("replayed without source")
}
hasStage := func(_ context.Context, k string) (bool, error) { return k == first.SourceKey, nil }
if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err != nil {
t.Fatal(err)
}
if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err == nil {
t.Fatal("replayed active job")
}
if c, err := s.Claim(ctx, first.Job, time.Minute); c != nil || err != nil {
t.Fatalf("old delivery %v %v", c, err)
}
replay := claim(t, publish(t))
if replay.Attempt != 3 || replay.JobID != first.JobID {
t.Fatalf("replay identity %+v", replay)
}
if ok, err := s.Fail(ctx, replay, "backend_unavailable", false, policy); !ok || err != nil {
t.Fatalf("fresh retry budget %v %v", ok, err)
}
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
last := claim(t, publish(t))
if ok, err := s.Fail(ctx, last, "source_missing", true, policy); !ok || err != nil {
t.Fatalf("permanent fail %v %v", ok, err)
}
var n int
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachment_replays WHERE actor='operator' AND previous_error='backend_unavailable'`).Scan(&n); err != nil || n != 1 {
t.Fatalf("replay audit %d %v", n, err)
}
})
t.Run("the previous version remains readable until the entire new result commits", func(t *testing.T) {
newJob(t)
old := claim(t, publish(t))
if ok, err := s.Complete(ctx, old, variants(old)); err != nil || !ok {
t.Fatalf("first publication %v %v", ok, err)
}
if err := s.Reprocess(ctx, old.AttachmentID, 2, exists); err != nil {
t.Fatal(err)
}
next := claim(t, publish(t))
if next.SourceKey != fmt.Sprintf("blobs/%x", old.Hash) {
t.Fatalf("version two source %s", next.SourceKey)
}
// Force an actual SQL failure on the second INSERT, after the first one succeeded.
exec(t, `ALTER TABLE attachment_variants ADD CONSTRAINT experiment_failure CHECK (pipeline_version<>2 OR variant<>'crop')`)
if _, err := s.Complete(ctx, next, variants(next)); err == nil {
t.Fatal("expected SQL failure on second variant")
}
exec(t, `ALTER TABLE attachment_variants DROP CONSTRAINT experiment_failure`)
var version, resultCount int
var status string
if err := db.Pool.QueryRow(ctx, `SELECT a.published_version,s.status,
(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2)
FROM attachment_states s JOIN attachments a USING(id)`).Scan(&version, &status, &resultCount); err != nil {
t.Fatal(err)
}
if version != 1 || status != "ready" || resultCount != 0 {
t.Fatalf("partial publication v%d status=%s rows=%d", version, status, resultCount)
}
if ok, err := s.Fail(ctx, next, "invalid_image", true, policy); err != nil || !ok {
t.Fatalf("permanent error %v %v", ok, err)
}
failed, err := s.Failed(ctx)
if err != nil || len(failed) != 1 || failed[0].Attempt != 1 || failed[0].ErrorCode != "invalid_image" {
t.Fatalf("permanent error not terminal %v %v", failed, err)
}
if err := db.Pool.QueryRow(ctx, `SELECT status FROM attachment_states`).Scan(&status); err != nil || status != "ready" {
t.Fatalf("failed reprocess hid original %s %v", status, err)
}
})
t.Run("migration preserves Lab4 message links and requeues known originals", func(t *testing.T) {
newJob(t)
provider, err := goose.NewProvider(goose.DialectPostgres, stdlib.OpenDBFromPool(db.Pool), migrations.FS)
if err != nil {
t.Fatal(err)
}
defer provider.Close()
if _, err := provider.DownTo(ctx, 3); err != nil {
t.Fatal(err)
}
exec(t, `UPDATE files SET status='ready'`)
exec(t, `WITH f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('cd',32),'hex'),10,'image/jpeg','failed') RETURNING hash)
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),chat_id,owner_id,'missing.jpg',f.hash,now() FROM attachments,f`)
exec(t, `WITH m AS (INSERT INTO messages(chat_id,id,sender_id,text,created_at) SELECT chat_id,gen_random_uuid(),owner_id,'old',now() FROM attachments LIMIT 1 RETURNING chat_id,id)
INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) SELECT m.chat_id,m.id,a.id,0 FROM m,attachments a WHERE a.filename='x.png'`)
if _, err := provider.Up(ctx); err != nil {
t.Fatal(err)
}
job := claim(t, publish(t))
if job.SourceKey != fmt.Sprintf("blobs/%x", bytes.Repeat([]byte{0xab}, 32)) {
t.Fatalf("migrated source %s", job.SourceKey)
}
var linked uuid.UUID
if err := db.Pool.QueryRow(ctx, `SELECT attachment_id FROM message_attachments`).Scan(&linked); err != nil || linked != job.AttachmentID {
t.Fatalf("changed link %s %v", linked, err)
}
failed, err := s.Failed(ctx)
if err != nil || len(failed) != 1 || failed[0].ErrorCode != "source_missing" {
t.Fatalf("migration failed registry %v %v", failed, err)
}
})
}
+79
View File
@@ -0,0 +1,79 @@
-- +goose Up
ALTER TABLE attachments
ADD COLUMN target_version integer NOT NULL DEFAULT 1 CHECK (target_version >= 1),
ADD COLUMN published_version integer CHECK (published_version BETWEEN 1 AND target_version);
-- A queued row with no publish timestamp is an outbox entry for this attempt.
-- attempt never decreases. The separate budget starts again only on operator replay.
CREATE TABLE attachment_jobs (
job_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
source_key text NOT NULL,
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
budget_attempt integer NOT NULL DEFAULT 1 CHECK (budget_attempt BETWEEN 1 AND attempt),
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
lease_token uuid,
lease_until timestamptz,
error_code text CHECK (error_code ~ '^[a-z][a-z0-9_]{0,63}$'),
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version),
CHECK ((state = 'processing') = (lease_token IS NOT NULL)),
CHECK ((state = 'processing') = (lease_until IS NOT NULL)),
CHECK (state <> 'failed' OR error_code IS NOT NULL)
);
CREATE INDEX attachment_jobs_outbox_idx ON attachment_jobs (available_at, job_id)
WHERE state='queued' AND published_at IS NULL;
CREATE INDEX attachment_jobs_recover_idx ON attachment_jobs (lease_until, job_id) WHERE state='processing';
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at, job_id) WHERE state='failed';
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL,
pipeline_version integer NOT NULL,
variant text NOT NULL CHECK (variant IN ('thumbnail', 'crop', 'optimized')),
object_key text NOT NULL,
mime_type text NOT NULL CHECK (mime_type = CASE WHEN variant='optimized' THEN 'image/webp' ELSE 'image/jpeg' END),
width integer NOT NULL CHECK (width > 0),
height integer NOT NULL CHECK (height > 0),
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
sha256 bytea NOT NULL CHECK (octet_length(sha256)=32),
PRIMARY KEY (attachment_id,pipeline_version,variant),
FOREIGN KEY (attachment_id,pipeline_version) REFERENCES attachment_jobs(attachment_id,pipeline_version)
);
CREATE TABLE attachment_replays (
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id),
attempt integer NOT NULL,
actor text NOT NULL CHECK (length(actor) BETWEEN 1 AND 128),
previous_error text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (job_id, attempt)
);
-- Published Lab 4 objects are valid inputs. A recorded missing original
-- stays diagnosable even before a worker is started.
INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code)
SELECT a.id,'blobs/'||encode(f.hash,'hex'),
CASE WHEN f.status='failed' THEN 'failed' ELSE 'queued' END,
CASE WHEN f.status='failed' THEN 'source_missing' END
FROM attachments a JOIN files f ON f.hash=a.file_hash;
CREATE VIEW attachment_states AS
SELECT a.id,
CASE WHEN a.published_version IS NULL THEN j.state
WHEN f.status='failed' THEN 'failed' ELSE 'ready' END AS status,
CASE WHEN a.published_version IS NOT NULL AND f.status='failed' THEN 'source_missing'
WHEN a.published_version IS NULL AND j.state='failed' THEN j.error_code END AS error_code,
j.updated_at
FROM attachments a JOIN files f ON f.hash=a.file_hash
JOIN attachment_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.target_version;
-- +goose Down
DROP VIEW attachment_states;
DROP TABLE attachment_replays;
DROP TABLE attachment_variants;
DROP TABLE attachment_jobs;
ALTER TABLE attachments DROP COLUMN published_version, DROP COLUMN target_version;