Сохраните задания, попытки и версии вложений в PostgreSQL
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type Job struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID uuid.UUID `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID uuid.UUID `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
var traceParent = regexp.MustCompile("^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||||
|
||||
func (j Job) Valid() bool {
|
||||
return j.SchemaVersion == 1 && j.Type == "attachment.process.v1" &&
|
||||
j.JobID != uuid.Nil && j.AttachmentID != uuid.Nil && j.PipelineVersion >= 1 &&
|
||||
j.Attempt >= 1 && !j.CreatedAt.IsZero() &&
|
||||
(j.Traceparent == "" || traceParent.MatchString(j.Traceparent)) && len(j.Tracestate) <= 512
|
||||
}
|
||||
|
||||
type Claim struct {
|
||||
Job
|
||||
Token uuid.UUID
|
||||
SourceKey string
|
||||
Hash []byte
|
||||
SizeBytes int64
|
||||
MIMEType string
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name, Key, MIMEType string
|
||||
Width, Height int
|
||||
SizeBytes int64
|
||||
SHA256 []byte
|
||||
}
|
||||
|
||||
func VariantKey(id uuid.UUID, version int, name string, hash []byte) string {
|
||||
return fmt.Sprintf("variants/%s/%d/%s/%x", id, version, name, hash)
|
||||
}
|
||||
|
||||
func validateVariants(c *Claim, variants []Variant) error {
|
||||
if len(variants) != 3 {
|
||||
return errors.New("three variants are required")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, v := range variants {
|
||||
mime := "image/jpeg"
|
||||
switch v.Name {
|
||||
case "thumbnail", "crop":
|
||||
case "optimized":
|
||||
mime = "image/webp"
|
||||
default:
|
||||
return errors.New("unknown variant")
|
||||
}
|
||||
if seen[v.Name] || v.MIMEType != mime || len(v.SHA256) != sha256.Size ||
|
||||
v.Key != VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, v.SHA256) ||
|
||||
v.Width <= 0 || v.Height <= 0 || v.SizeBytes <= 0 {
|
||||
return errors.New("invalid variant metadata")
|
||||
}
|
||||
seen[v.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RetryPolicy struct {
|
||||
MaxAttempts int
|
||||
BaseDelay, MaxDelay time.Duration
|
||||
}
|
||||
|
||||
func (p RetryPolicy) validate() error {
|
||||
if p.MaxAttempts < 1 || p.BaseDelay < time.Microsecond || p.MaxDelay < p.BaseDelay {
|
||||
return errors.New("invalid pipeline retry policy")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Equal jitter keeps a minimum delay while distributing simultaneous failures.
|
||||
func (p RetryPolicy) delay(attempt int) time.Duration {
|
||||
limit := p.BaseDelay
|
||||
for i := 1; i < attempt && limit < p.MaxDelay; i++ {
|
||||
if limit > p.MaxDelay/2 {
|
||||
limit = p.MaxDelay
|
||||
} else {
|
||||
limit *= 2
|
||||
}
|
||||
}
|
||||
return limit/2 + time.Duration(rand.Int64N(int64(limit-limit/2)+1))
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/platform/apperr"
|
||||
)
|
||||
|
||||
type FailedJob struct {
|
||||
JobID uuid.UUID `json:"job_id"`
|
||||
AttachmentID uuid.UUID `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (s *Store) Failed(ctx context.Context) ([]FailedJob, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,error_code,updated_at
|
||||
FROM attachment_jobs WHERE state='failed' ORDER BY updated_at,job_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgx.CollectRows(rows, pgx.RowToStructByPos[FailedJob])
|
||||
}
|
||||
|
||||
// Lock the attachment before checking the current target, its source or an S3
|
||||
// object. This serializes replay and version changes with source collection.
|
||||
func lockTarget(ctx context.Context, tx pgx.Tx, id uuid.UUID) (int, []byte, error) {
|
||||
var version int
|
||||
var hash []byte
|
||||
err := tx.QueryRow(ctx, `SELECT target_version,file_hash FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&version, &hash)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
err = apperr.NotFound("attachment not found")
|
||||
}
|
||||
return version, hash, err
|
||||
}
|
||||
|
||||
func availableSource(ctx context.Context, preferred, fallback string, exists func(context.Context, string) (bool, error)) (string, error) {
|
||||
ok, err := exists(ctx, preferred)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ok {
|
||||
return preferred, nil
|
||||
}
|
||||
if fallback != preferred {
|
||||
ok, err = exists(ctx, fallback)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ok {
|
||||
return fallback, nil
|
||||
}
|
||||
}
|
||||
return "", apperr.Conflict("processing source is missing")
|
||||
}
|
||||
|
||||
func (s *Store) Reprocess(ctx context.Context, id uuid.UUID, version int, exists func(context.Context, string) (bool, error)) error {
|
||||
if version != 1 && version != 2 {
|
||||
return apperr.Invalid("unsupported pipeline version")
|
||||
}
|
||||
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
current, hash, err := lockTarget(ctx, tx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if version <= current {
|
||||
return apperr.Conflict("target version must increase")
|
||||
}
|
||||
var source string
|
||||
if err := tx.QueryRow(ctx, `SELECT source_key FROM attachment_jobs
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2 FOR UPDATE`, id, current).Scan(&source); err != nil {
|
||||
return err
|
||||
}
|
||||
source, err = availableSource(ctx, attachment.BlobKey(hash), source, exists)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Keep committed result rows immutable and readable until the switch.
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='superseded',lease_token=NULL,lease_until=NULL,updated_at=clock_timestamp()
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2 AND state<>'ready'`, id, current); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key)
|
||||
VALUES ($1,$2,$3)`, id, version, source); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET target_version=$2 WHERE id=$1`, id, version)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) Replay(ctx context.Context, id uuid.UUID, actor string, exists func(context.Context, string) (bool, error)) error {
|
||||
actor = strings.TrimSpace(actor)
|
||||
if actor == "" || len(actor) > 128 {
|
||||
return apperr.Invalid("replay requires an operator name of up to 128 bytes")
|
||||
}
|
||||
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
current, hash, err := lockTarget(ctx, tx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var jobID uuid.UUID
|
||||
var source, code string
|
||||
var attempt int
|
||||
err = tx.QueryRow(ctx, `SELECT job_id,source_key,error_code,attempt FROM attachment_jobs
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2 AND state='failed' FOR UPDATE`, id, current).
|
||||
Scan(&jobID, &source, &code, &attempt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return apperr.Conflict("current job is not failed")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, err = availableSource(ctx, source, attachment.BlobKey(hash), exists)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,attempt,actor,previous_error) VALUES ($1,$2,$3,$4)`,
|
||||
jobID, attempt+1, actor, code); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempt=attempt+1,budget_attempt=1,source_key=$2,
|
||||
published_at=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),error_code=NULL WHERE job_id=$1`, jobID, source)
|
||||
return err
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"regexp"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
var errorCodePattern = regexp.MustCompile("^[a-z][a-z0-9_]{0,63}$")
|
||||
|
||||
func (s *Store) Fail(ctx context.Context, c *Claim, code string, permanent bool, policy RetryPolicy) (bool, error) {
|
||||
if err := policy.validate(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !errorCodePattern.MatchString(code) {
|
||||
return false, errors.New("invalid failure code")
|
||||
}
|
||||
var changed bool
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
var budget int
|
||||
err := tx.QueryRow(ctx, `SELECT budget_attempt FROM attachment_jobs
|
||||
WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3 AND attempt=$4
|
||||
AND state='processing' AND lease_token=$5 AND lease_until>clock_timestamp() FOR UPDATE`,
|
||||
c.JobID, c.AttachmentID, c.PipelineVersion, c.Attempt, c.Token).Scan(&budget)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = finishAttempt(ctx, tx, c.JobID, budget, code, permanent, false, policy)
|
||||
return err
|
||||
})
|
||||
return changed && err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error) {
|
||||
if err := policy.validate(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
var changed bool
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
var id uuid.UUID
|
||||
var budget int
|
||||
err := tx.QueryRow(ctx, `SELECT job_id,budget_attempt FROM attachment_jobs
|
||||
WHERE state='processing' AND lease_until<=clock_timestamp()
|
||||
ORDER BY lease_until,job_id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&id, &budget)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = finishAttempt(ctx, tx, id, budget, "lease_expired", false, true, policy)
|
||||
return err
|
||||
})
|
||||
return changed && err == nil, err
|
||||
}
|
||||
|
||||
// The caller holds the job row. Only the recovery path may advance an expired
|
||||
// attempt. A worker whose lease elapsed during its SQL call has lost ownership.
|
||||
func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, code string, permanent, expired bool, p RetryPolicy) (bool, error) {
|
||||
retry := !permanent && budget < p.MaxAttempts
|
||||
tag, err := tx.Exec(ctx, `UPDATE attachment_jobs
|
||||
SET state=CASE WHEN $3 THEN 'queued' ELSE 'failed' END,
|
||||
attempt=attempt+$3::boolean::int, budget_attempt=budget_attempt+$3::boolean::int,
|
||||
available_at=clock_timestamp()+$4::bigint*interval '1 microsecond',
|
||||
published_at=NULL, lease_token=NULL, lease_until=NULL, error_code=$2, updated_at=clock_timestamp()
|
||||
WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, code, retry, p.delay(budget).Microseconds(), expired)
|
||||
return tag.RowsAffected() == 1, err
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
)
|
||||
|
||||
type Store struct{ pool *pgxpool.Pool }
|
||||
|
||||
func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
|
||||
|
||||
// The callback waits for a routed publisher confirm with a finite timeout.
|
||||
// Losing that confirm or the SQL commit leaves the same envelope deliverable.
|
||||
func (s *Store) PublishOne(ctx context.Context, publish func(context.Context, Job) error) (bool, error) {
|
||||
var found bool
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
j := Job{SchemaVersion: 1, Type: "attachment.process.v1"}
|
||||
err := tx.QueryRow(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,created_at
|
||||
FROM attachment_jobs WHERE state='queued' AND published_at IS NULL AND available_at<=clock_timestamp()
|
||||
ORDER BY available_at,job_id LIMIT 1 FOR UPDATE SKIP LOCKED`).
|
||||
Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found = true
|
||||
if err := publish(ctx, j); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET published_at=clock_timestamp()
|
||||
WHERE job_id=$1 AND attempt=$2 AND state='queued'`, j.JobID, j.Attempt)
|
||||
return err
|
||||
})
|
||||
return found, err
|
||||
}
|
||||
|
||||
// Claim checks the envelope against SQL and sources all object metadata from SQL.
|
||||
// A processing duplicate can be acknowledged: lease recovery is independent
|
||||
// of broker redelivery and persists a fresh outbox intent after worker death.
|
||||
func (s *Store) Claim(ctx context.Context, j Job, lease time.Duration) (*Claim, error) {
|
||||
if !j.Valid() || lease < time.Microsecond {
|
||||
return nil, errors.New("invalid pipeline job or lease")
|
||||
}
|
||||
c := Claim{Job: j, Token: uuid.New()}
|
||||
err := s.pool.QueryRow(ctx, `UPDATE attachment_jobs j SET state='processing', lease_token=$6,
|
||||
lease_until=clock_timestamp()+$7::bigint*interval '1 microsecond', updated_at=clock_timestamp()
|
||||
FROM attachments a JOIN files f ON f.hash=a.file_hash
|
||||
WHERE j.job_id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.attempt=$4 AND j.created_at=$5
|
||||
AND a.id=j.attachment_id AND a.target_version=j.pipeline_version
|
||||
AND j.state='queued' AND j.available_at<=clock_timestamp()
|
||||
RETURNING j.source_key,f.hash,f.size_bytes,f.mime_type`,
|
||||
j.JobID, j.AttachmentID, j.PipelineVersion, j.Attempt, j.CreatedAt, c.Token, lease.Microseconds()).
|
||||
Scan(&c.SourceKey, &c.Hash, &c.SizeBytes, &c.MIMEType)
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
var incomplete bool
|
||||
err = s.pool.QueryRow(ctx, `SELECT EXISTS (
|
||||
SELECT 1 FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
||||
WHERE j.job_id=$1 AND j.state='ready' AND
|
||||
(coalesce(a.published_version,0)<j.pipeline_version OR 3<>(
|
||||
SELECT count(*) FROM attachment_variants v WHERE v.attachment_id=a.id AND v.pipeline_version=j.pipeline_version))
|
||||
)`, j.JobID).Scan(&incomplete)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if incomplete {
|
||||
return nil, errors.New("ready job has incomplete durable result")
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var errLostClaim = errors.New("claim expired or superseded")
|
||||
|
||||
func (s *Store) Complete(ctx context.Context, c *Claim, variants []Variant) (bool, error) {
|
||||
if err := validateVariants(c, variants); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if err := attachment.LockHash(ctx, tx, c.Hash); err != nil {
|
||||
return err
|
||||
}
|
||||
// Lock the attachment before the job, in the same order as operator commands.
|
||||
tag, err := tx.Exec(ctx, `SELECT 1 FROM attachments WHERE id=$1 AND target_version=$2 FOR UPDATE`, c.AttachmentID, c.PipelineVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errLostClaim
|
||||
}
|
||||
tag, err = tx.Exec(ctx, `SELECT 1 FROM attachment_jobs WHERE job_id=$1 AND attachment_id=$2 AND pipeline_version=$3
|
||||
AND state='processing' AND lease_token=$4 AND lease_until>clock_timestamp() FOR UPDATE`,
|
||||
c.JobID, c.AttachmentID, c.PipelineVersion, c.Token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errLostClaim
|
||||
}
|
||||
for _, v := range variants {
|
||||
_, err := tx.Exec(ctx, `INSERT INTO attachment_variants
|
||||
(attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
|
||||
c.AttachmentID, c.PipelineVersion, v.Name, v.Key, v.MIMEType, v.Width, v.Height, v.SizeBytes, v.SHA256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE files SET status='ready' WHERE hash=(SELECT file_hash FROM attachments WHERE id=$1)`, c.AttachmentID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET published_version=$2 WHERE id=$1`, c.AttachmentID, c.PipelineVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
// Recheck DB time after every operation that could have waited on a lock.
|
||||
tag, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='ready',lease_until=NULL,lease_token=NULL,
|
||||
error_code=NULL,updated_at=clock_timestamp() WHERE job_id=$1 AND lease_until>clock_timestamp()`, c.JobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return errLostClaim
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if errors.Is(err, errLostClaim) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/pressly/goose/v3"
|
||||
|
||||
"messenger/internal/testenv"
|
||||
"messenger/migrations"
|
||||
)
|
||||
|
||||
func TestPipelineTransitions(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
db, err := testenv.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(db.Close)
|
||||
s := NewStore(db.Pool)
|
||||
exists := func(context.Context, string) (bool, error) { return true, nil }
|
||||
policy := RetryPolicy{MaxAttempts: 2, BaseDelay: time.Millisecond, MaxDelay: time.Second}
|
||||
exec := func(t *testing.T, query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.Pool.Exec(ctx, query, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
newJob := func(t *testing.T) {
|
||||
t.Helper()
|
||||
db.Reset(t)
|
||||
id := uuid.New()
|
||||
exec(t, `WITH u AS (INSERT INTO users VALUES (gen_random_uuid(),'test','Test User','hash',now()) RETURNING id),
|
||||
c AS (INSERT INTO chats VALUES (gen_random_uuid(),'chat',now()) RETURNING id),
|
||||
f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('ab',32),'hex'),10,'image/png','pending') RETURNING hash)
|
||||
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at)
|
||||
SELECT $1,c.id,u.id,'x.png',f.hash,now() FROM u,c,f`, id)
|
||||
exec(t, `INSERT INTO attachment_jobs(attachment_id,source_key) VALUES ($1,$2)`, id, "staging/"+id.String())
|
||||
}
|
||||
publish := func(t *testing.T) Job {
|
||||
t.Helper()
|
||||
var j Job
|
||||
ok, err := s.PublishOne(ctx, func(_ context.Context, v Job) error { j = v; return nil })
|
||||
if err != nil || !ok || !j.Valid() {
|
||||
t.Fatalf("publish: %+v, %v, %v", j, ok, err)
|
||||
}
|
||||
return j
|
||||
}
|
||||
claim := func(t *testing.T, j Job) *Claim {
|
||||
t.Helper()
|
||||
c, err := s.Claim(ctx, j, time.Minute)
|
||||
if err != nil || c == nil {
|
||||
t.Fatalf("claim: %v %v", c, err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
variants := func(c *Claim) []Variant {
|
||||
var vv []Variant
|
||||
for i, name := range []string{"thumbnail", "crop", "optimized"} {
|
||||
hash := bytes.Repeat([]byte{byte(c.PipelineVersion*10 + i)}, 32)
|
||||
mime := "image/jpeg"
|
||||
if name == "optimized" {
|
||||
mime = "image/webp"
|
||||
}
|
||||
vv = append(vv, Variant{Name: name, Key: fmt.Sprintf("variants/%s/%d/%s/%x", c.AttachmentID, c.PipelineVersion, name, hash), Width: 128, Height: 128, SizeBytes: 4, MIMEType: mime, SHA256: hash})
|
||||
}
|
||||
return vv
|
||||
}
|
||||
t.Run("unconfirmed publication is retried and concurrent publishers skip locked work", func(t *testing.T) {
|
||||
newJob(t)
|
||||
var seen Job
|
||||
confirmLost := errors.New("confirm lost")
|
||||
_, err := s.PublishOne(ctx, func(_ context.Context, j Job) error {
|
||||
seen = j
|
||||
ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("double publish") })
|
||||
if err != nil || ok {
|
||||
t.Fatalf("second publisher %v %v", ok, err)
|
||||
}
|
||||
return confirmLost
|
||||
})
|
||||
if !errors.Is(err, confirmLost) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := publish(t)
|
||||
if j != seen {
|
||||
t.Fatalf("redelivery changed envelope: %+v %+v", seen, j)
|
||||
}
|
||||
ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("duplicate") })
|
||||
if ok || err != nil {
|
||||
t.Fatalf("confirmed work republished: %v %v", ok, err)
|
||||
}
|
||||
})
|
||||
t.Run("backoff gates publication and consumption of the next attempt", func(t *testing.T) {
|
||||
newJob(t)
|
||||
first := claim(t, publish(t))
|
||||
policy := RetryPolicy{MaxAttempts: 3, BaseDelay: 4 * time.Second, MaxDelay: 5 * time.Second}
|
||||
if ok, err := s.Fail(ctx, first, "storage_timeout", false, policy); !ok || err != nil {
|
||||
t.Fatalf("failed %v %v", ok, err)
|
||||
}
|
||||
var seconds float64
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if seconds < 1.99 || seconds > 4.01 {
|
||||
t.Fatalf("first retry delay %.3f seconds", seconds)
|
||||
}
|
||||
next := first.Job
|
||||
next.Attempt++
|
||||
if c, err := s.Claim(ctx, next, time.Minute); err != nil || c != nil {
|
||||
t.Fatalf("premature claim %v %v", c, err)
|
||||
}
|
||||
if ok, err := s.PublishOne(ctx, func(context.Context, Job) error { return errors.New("premature publish") }); err != nil || ok {
|
||||
t.Fatalf("premature publish %v %v", ok, err)
|
||||
}
|
||||
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
|
||||
second := claim(t, publish(t))
|
||||
if ok, err := s.Fail(ctx, second, "storage_timeout", false, policy); !ok || err != nil {
|
||||
t.Fatalf("second failure %v %v", ok, err)
|
||||
}
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if seconds < 2.49 || seconds > 5.01 {
|
||||
t.Fatalf("capped second retry delay %.3f seconds", seconds)
|
||||
}
|
||||
})
|
||||
t.Run("parallel duplicates and version change before commit", func(t *testing.T) {
|
||||
newJob(t)
|
||||
j := publish(t)
|
||||
var wg sync.WaitGroup
|
||||
claims := make(chan *Claim, 8)
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
c, err := s.Claim(ctx, j, time.Minute)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if c != nil {
|
||||
claims <- c
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(claims)
|
||||
if len(claims) != 1 {
|
||||
t.Fatalf("%d workers claimed one job", len(claims))
|
||||
}
|
||||
old := <-claims
|
||||
if old.SourceKey != "staging/"+j.AttachmentID.String() || old.SizeBytes != 10 || !bytes.Equal(old.Hash, bytes.Repeat([]byte{0xab}, 32)) {
|
||||
t.Fatalf("source metadata: %+v", old)
|
||||
}
|
||||
stageOnly := func(_ context.Context, key string) (bool, error) { return key == old.SourceKey, nil }
|
||||
// A different attachment can have published the same digest. That row's
|
||||
// status cannot prove the object is still there when the target changes.
|
||||
exec(t, `UPDATE files SET status='ready'`)
|
||||
if err := s.Reprocess(ctx, j.AttachmentID, 2, stageOnly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := claim(t, publish(t))
|
||||
if current.SourceKey != old.SourceKey {
|
||||
t.Fatalf("available staging ignored in favor of missing CAS: %s", current.SourceKey)
|
||||
}
|
||||
results := variants(current)
|
||||
if _, err := s.Complete(ctx, current, results[:2]); err == nil {
|
||||
t.Fatal("published incomplete result")
|
||||
}
|
||||
if ok, err := s.Complete(ctx, current, results); err != nil || !ok {
|
||||
t.Fatalf("complete %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.Complete(ctx, old, variants(old)); err != nil || ok {
|
||||
t.Fatalf("stale commit %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, old, "source_missing", true, policy); err != nil || ok {
|
||||
t.Fatalf("stale failure %v %v", ok, err)
|
||||
}
|
||||
for _, dup := range []Job{current.Job, j, current.Job} {
|
||||
if c, err := s.Claim(ctx, dup, time.Minute); err != nil || c != nil {
|
||||
t.Fatalf("duplicate %v %v", c, err)
|
||||
}
|
||||
}
|
||||
var version, count int
|
||||
var digest []byte
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT a.published_version,
|
||||
(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id),
|
||||
(SELECT sha256 FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2 AND variant='optimized')
|
||||
FROM attachments a WHERE a.id=$1`, j.AttachmentID).Scan(&version, &count, &digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != 2 || count != 3 || !bytes.Equal(digest, results[2].SHA256) {
|
||||
t.Fatalf("published v%d count %d hash %x", version, count, digest)
|
||||
}
|
||||
exec(t, `DELETE FROM attachment_variants WHERE variant='crop'`)
|
||||
if _, err := s.Claim(ctx, current.Job, time.Minute); err == nil {
|
||||
t.Fatal("ready duplicate acknowledged without complete result")
|
||||
}
|
||||
})
|
||||
t.Run("dead lease and retry exhaustion then audited replay", func(t *testing.T) {
|
||||
newJob(t)
|
||||
first := claim(t, publish(t))
|
||||
exec(t, `UPDATE attachment_jobs SET lease_until=clock_timestamp()-interval '1 second'`)
|
||||
if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok {
|
||||
t.Fatalf("expired commit %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, first, "backend_unavailable", false, policy); err != nil || ok {
|
||||
t.Fatalf("expired fail %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.RecoverOne(ctx, policy); err != nil || !ok {
|
||||
t.Fatalf("recover %v %v", ok, err)
|
||||
}
|
||||
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
|
||||
second := claim(t, publish(t))
|
||||
if first.JobID != second.JobID || first.Attempt != 1 || second.Attempt != 2 || first.Token == second.Token {
|
||||
t.Fatalf("attempt identity %+v %+v", first, second)
|
||||
}
|
||||
if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok {
|
||||
t.Fatalf("lost lease commit %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, second, "backend_unavailable", false, policy); err != nil || !ok {
|
||||
t.Fatalf("second error %v %v", ok, err)
|
||||
}
|
||||
failed, err := s.Failed(ctx)
|
||||
if err != nil || len(failed) != 1 {
|
||||
t.Fatalf("failed %v %v", failed, err)
|
||||
}
|
||||
absent := func(context.Context, string) (bool, error) { return false, nil }
|
||||
if err := s.Replay(ctx, first.AttachmentID, "test", absent); err == nil {
|
||||
t.Fatal("replayed without source")
|
||||
}
|
||||
hasStage := func(_ context.Context, k string) (bool, error) { return k == first.SourceKey, nil }
|
||||
if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Replay(ctx, first.AttachmentID, "operator", hasStage); err == nil {
|
||||
t.Fatal("replayed active job")
|
||||
}
|
||||
if c, err := s.Claim(ctx, first.Job, time.Minute); c != nil || err != nil {
|
||||
t.Fatalf("old delivery %v %v", c, err)
|
||||
}
|
||||
replay := claim(t, publish(t))
|
||||
if replay.Attempt != 3 || replay.JobID != first.JobID {
|
||||
t.Fatalf("replay identity %+v", replay)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, replay, "backend_unavailable", false, policy); !ok || err != nil {
|
||||
t.Fatalf("fresh retry budget %v %v", ok, err)
|
||||
}
|
||||
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
|
||||
last := claim(t, publish(t))
|
||||
if ok, err := s.Fail(ctx, last, "source_missing", true, policy); !ok || err != nil {
|
||||
t.Fatalf("permanent fail %v %v", ok, err)
|
||||
}
|
||||
var n int
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT count(*) FROM attachment_replays WHERE actor='operator' AND previous_error='backend_unavailable'`).Scan(&n); err != nil || n != 1 {
|
||||
t.Fatalf("replay audit %d %v", n, err)
|
||||
}
|
||||
})
|
||||
t.Run("the previous version remains readable until the entire new result commits", func(t *testing.T) {
|
||||
newJob(t)
|
||||
old := claim(t, publish(t))
|
||||
if ok, err := s.Complete(ctx, old, variants(old)); err != nil || !ok {
|
||||
t.Fatalf("first publication %v %v", ok, err)
|
||||
}
|
||||
if err := s.Reprocess(ctx, old.AttachmentID, 2, exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := claim(t, publish(t))
|
||||
if next.SourceKey != fmt.Sprintf("blobs/%x", old.Hash) {
|
||||
t.Fatalf("version two source %s", next.SourceKey)
|
||||
}
|
||||
// Force an actual SQL failure on the second INSERT, after the first one succeeded.
|
||||
exec(t, `ALTER TABLE attachment_variants ADD CONSTRAINT experiment_failure CHECK (pipeline_version<>2 OR variant<>'crop')`)
|
||||
if _, err := s.Complete(ctx, next, variants(next)); err == nil {
|
||||
t.Fatal("expected SQL failure on second variant")
|
||||
}
|
||||
exec(t, `ALTER TABLE attachment_variants DROP CONSTRAINT experiment_failure`)
|
||||
var version, resultCount int
|
||||
var status string
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT a.published_version,s.status,
|
||||
(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id AND pipeline_version=2)
|
||||
FROM attachment_states s JOIN attachments a USING(id)`).Scan(&version, &status, &resultCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != 1 || status != "ready" || resultCount != 0 {
|
||||
t.Fatalf("partial publication v%d status=%s rows=%d", version, status, resultCount)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, next, "invalid_image", true, policy); err != nil || !ok {
|
||||
t.Fatalf("permanent error %v %v", ok, err)
|
||||
}
|
||||
failed, err := s.Failed(ctx)
|
||||
if err != nil || len(failed) != 1 || failed[0].Attempt != 1 || failed[0].ErrorCode != "invalid_image" {
|
||||
t.Fatalf("permanent error not terminal %v %v", failed, err)
|
||||
}
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT status FROM attachment_states`).Scan(&status); err != nil || status != "ready" {
|
||||
t.Fatalf("failed reprocess hid original %s %v", status, err)
|
||||
}
|
||||
})
|
||||
t.Run("migration preserves Lab4 message links and requeues known originals", func(t *testing.T) {
|
||||
newJob(t)
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, stdlib.OpenDBFromPool(db.Pool), migrations.FS)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer provider.Close()
|
||||
if _, err := provider.DownTo(ctx, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exec(t, `UPDATE files SET status='ready'`)
|
||||
exec(t, `WITH f AS (INSERT INTO files(hash,size_bytes,mime_type,status) VALUES (decode(repeat('cd',32),'hex'),10,'image/jpeg','failed') RETURNING hash)
|
||||
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) SELECT gen_random_uuid(),chat_id,owner_id,'missing.jpg',f.hash,now() FROM attachments,f`)
|
||||
exec(t, `WITH m AS (INSERT INTO messages(chat_id,id,sender_id,text,created_at) SELECT chat_id,gen_random_uuid(),owner_id,'old',now() FROM attachments LIMIT 1 RETURNING chat_id,id)
|
||||
INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) SELECT m.chat_id,m.id,a.id,0 FROM m,attachments a WHERE a.filename='x.png'`)
|
||||
if _, err := provider.Up(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := claim(t, publish(t))
|
||||
if job.SourceKey != fmt.Sprintf("blobs/%x", bytes.Repeat([]byte{0xab}, 32)) {
|
||||
t.Fatalf("migrated source %s", job.SourceKey)
|
||||
}
|
||||
var linked uuid.UUID
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT attachment_id FROM message_attachments`).Scan(&linked); err != nil || linked != job.AttachmentID {
|
||||
t.Fatalf("changed link %s %v", linked, err)
|
||||
}
|
||||
failed, err := s.Failed(ctx)
|
||||
if err != nil || len(failed) != 1 || failed[0].ErrorCode != "source_missing" {
|
||||
t.Fatalf("migration failed registry %v %v", failed, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN target_version integer NOT NULL DEFAULT 1 CHECK (target_version >= 1),
|
||||
ADD COLUMN published_version integer CHECK (published_version BETWEEN 1 AND target_version);
|
||||
|
||||
-- A queued row with no publish timestamp is an outbox entry for this attempt.
|
||||
-- attempt never decreases. The separate budget starts again only on operator replay.
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
|
||||
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
|
||||
source_key text NOT NULL,
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
budget_attempt integer NOT NULL DEFAULT 1 CHECK (budget_attempt BETWEEN 1 AND attempt),
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
lease_token uuid,
|
||||
lease_until timestamptz,
|
||||
error_code text CHECK (error_code ~ '^[a-z][a-z0-9_]{0,63}$'),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (attachment_id, pipeline_version),
|
||||
CHECK ((state = 'processing') = (lease_token IS NOT NULL)),
|
||||
CHECK ((state = 'processing') = (lease_until IS NOT NULL)),
|
||||
CHECK (state <> 'failed' OR error_code IS NOT NULL)
|
||||
);
|
||||
CREATE INDEX attachment_jobs_outbox_idx ON attachment_jobs (available_at, job_id)
|
||||
WHERE state='queued' AND published_at IS NULL;
|
||||
CREATE INDEX attachment_jobs_recover_idx ON attachment_jobs (lease_until, job_id) WHERE state='processing';
|
||||
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at, job_id) WHERE state='failed';
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL,
|
||||
pipeline_version integer NOT NULL,
|
||||
variant text NOT NULL CHECK (variant IN ('thumbnail', 'crop', 'optimized')),
|
||||
object_key text NOT NULL,
|
||||
mime_type text NOT NULL CHECK (mime_type = CASE WHEN variant='optimized' THEN 'image/webp' ELSE 'image/jpeg' END),
|
||||
width integer NOT NULL CHECK (width > 0),
|
||||
height integer NOT NULL CHECK (height > 0),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
|
||||
sha256 bytea NOT NULL CHECK (octet_length(sha256)=32),
|
||||
PRIMARY KEY (attachment_id,pipeline_version,variant),
|
||||
FOREIGN KEY (attachment_id,pipeline_version) REFERENCES attachment_jobs(attachment_id,pipeline_version)
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_replays (
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id),
|
||||
attempt integer NOT NULL,
|
||||
actor text NOT NULL CHECK (length(actor) BETWEEN 1 AND 128),
|
||||
previous_error text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (job_id, attempt)
|
||||
);
|
||||
|
||||
-- Published Lab 4 objects are valid inputs. A recorded missing original
|
||||
-- stays diagnosable even before a worker is started.
|
||||
INSERT INTO attachment_jobs(attachment_id,source_key,state,error_code)
|
||||
SELECT a.id,'blobs/'||encode(f.hash,'hex'),
|
||||
CASE WHEN f.status='failed' THEN 'failed' ELSE 'queued' END,
|
||||
CASE WHEN f.status='failed' THEN 'source_missing' END
|
||||
FROM attachments a JOIN files f ON f.hash=a.file_hash;
|
||||
|
||||
CREATE VIEW attachment_states AS
|
||||
SELECT a.id,
|
||||
CASE WHEN a.published_version IS NULL THEN j.state
|
||||
WHEN f.status='failed' THEN 'failed' ELSE 'ready' END AS status,
|
||||
CASE WHEN a.published_version IS NOT NULL AND f.status='failed' THEN 'source_missing'
|
||||
WHEN a.published_version IS NULL AND j.state='failed' THEN j.error_code END AS error_code,
|
||||
j.updated_at
|
||||
FROM attachments a JOIN files f ON f.hash=a.file_hash
|
||||
JOIN attachment_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.target_version;
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW attachment_states;
|
||||
DROP TABLE attachment_replays;
|
||||
DROP TABLE attachment_variants;
|
||||
DROP TABLE attachment_jobs;
|
||||
ALTER TABLE attachments DROP COLUMN published_version, DROP COLUMN target_version;
|
||||
Reference in New Issue
Block a user