Разделите настройки команд worker и сохраняйте причину сбоя
This commit is contained in:
+104
-66
@@ -5,11 +5,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -37,61 +35,110 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
const usage = "use failed, replay <id> <operator>, reprocess <id> <version>, or duplicate <id> <version> <attempt>"
|
||||
|
||||
type command struct {
|
||||
name, actor, barrierDir string
|
||||
id uuid.UUID
|
||||
version, attempt int
|
||||
}
|
||||
|
||||
func parseCommand(args []string) (command, error) {
|
||||
var c command
|
||||
if len(args) == 0 {
|
||||
return c, nil
|
||||
}
|
||||
if dir, ok := strings.CutPrefix(args[0], "--barriers="); ok && len(args) == 1 && dir != "" {
|
||||
c.barrierDir = dir
|
||||
return c, nil
|
||||
}
|
||||
c.name = args[0]
|
||||
switch c.name {
|
||||
case "--prepare-large-png", "failed":
|
||||
if len(args) == 1 {
|
||||
return c, nil
|
||||
}
|
||||
return c, errors.New(usage)
|
||||
case "replay", "reprocess":
|
||||
if len(args) != 3 {
|
||||
return c, errors.New(usage)
|
||||
}
|
||||
case "duplicate":
|
||||
if len(args) != 4 {
|
||||
return c, errors.New(usage)
|
||||
}
|
||||
default:
|
||||
return c, errors.New(usage)
|
||||
}
|
||||
var err error
|
||||
c.id, err = uuid.Parse(args[1])
|
||||
if err != nil {
|
||||
return c, errors.New("invalid attachment UUID")
|
||||
}
|
||||
if c.name == "replay" {
|
||||
c.actor = strings.TrimSpace(args[2])
|
||||
if c.actor == "" || len(c.actor) > 128 {
|
||||
return c, errors.New("replay requires an operator name of up to 128 bytes")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
c.version, err = strconv.Atoi(args[2])
|
||||
if err != nil || c.version < 1 {
|
||||
return c, errors.New("invalid pipeline version")
|
||||
}
|
||||
if c.name == "duplicate" {
|
||||
c.attempt, err = strconv.Atoi(args[3])
|
||||
if err != nil || c.attempt < 1 {
|
||||
return c, errors.New("invalid attempt number")
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string) error {
|
||||
if len(args) == 1 && args[0] == "--prepare-large-png" {
|
||||
cmd, err := parseCommand(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cmd.name == "--prepare-large-png" {
|
||||
return imaging.PrepareLargePNG(os.Stdin)
|
||||
}
|
||||
if cmd.name != "" {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
}
|
||||
db, err := config.LoadDatabaseURL(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg, err := config.LoadJobs(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s3, err := config.LoadS3(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobs, err := s3store.New(s3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := postgres.NewPool(ctx, db, 5)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
store := pipeline.NewStore(pool)
|
||||
w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()}
|
||||
if len(args) == 1 && strings.HasPrefix(args[0], "--barriers=") {
|
||||
dir := strings.TrimPrefix(args[0], "--barriers=")
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
w.Checkpoint = processing.FileBarrier(dir)
|
||||
args = nil
|
||||
}
|
||||
if len(args) == 0 {
|
||||
w.Run(ctx)
|
||||
return nil
|
||||
}
|
||||
op, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
if len(args) == 1 && args[0] == "failed" {
|
||||
jobs, err := store.Failed(op)
|
||||
switch cmd.name {
|
||||
case "failed":
|
||||
jobs, err := store.Failed(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(jobs)
|
||||
case "duplicate":
|
||||
url, err := config.LoadBrokerURL(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return duplicate(ctx, pool, url, cmd.id, cmd.version, cmd.attempt)
|
||||
}
|
||||
const usage = "use failed, replay <id> <operator>, reprocess <id> <version>, or duplicate <id> <version> <attempt>"
|
||||
if len(args) < 3 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
id, err := uuid.Parse(args[1])
|
||||
s3, err := config.LoadS3(os.LookupEnv)
|
||||
if err != nil {
|
||||
return errors.New("invalid attachment UUID")
|
||||
return err
|
||||
}
|
||||
blobs, err := s3store.New(s3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
exists := func(ctx context.Context, key string) (bool, error) {
|
||||
_, err := blobs.Stat(ctx, key)
|
||||
@@ -100,37 +147,28 @@ func run(ctx context.Context, args []string) error {
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
switch args[0] {
|
||||
switch cmd.name {
|
||||
case "replay":
|
||||
if len(args) != 3 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
return store.Replay(op, id, args[2], exists)
|
||||
return store.Replay(ctx, cmd.id, cmd.actor, exists)
|
||||
case "reprocess":
|
||||
if len(args) != 3 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
v, err := strconv.Atoi(args[2])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return store.Reprocess(op, id, v, exists)
|
||||
case "duplicate":
|
||||
if len(args) != 4 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
version, err := strconv.Atoi(args[2])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attempt, err := strconv.Atoi(args[3])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return duplicate(op, pool, cfg.URL, id, version, attempt)
|
||||
default:
|
||||
return errors.New(usage)
|
||||
return store.Reprocess(ctx, cmd.id, cmd.version, exists)
|
||||
}
|
||||
cfg, err := config.LoadJobs(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()}
|
||||
if cmd.barrierDir != "" {
|
||||
if err := os.MkdirAll(cmd.barrierDir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
w.Checkpoint = processing.FileBarrier(cmd.barrierDir)
|
||||
}
|
||||
if err := imaging.CleanScratch(); err != nil {
|
||||
return err
|
||||
}
|
||||
w.Run(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
// duplicate sends a previously issued envelope through the actual broker. It
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"messenger/internal/testenv"
|
||||
)
|
||||
|
||||
func TestInvalidCommandBeforeConfig(t *testing.T) {
|
||||
t.Setenv("DATABASE_URL_FILE", "")
|
||||
for _, args := range [][]string{{"unknown"}, {"replay", "bad-id", "admin"}, {"duplicate", "00000000-0000-4000-8000-000000000001", "1", "invalid"}} {
|
||||
err := run(t.Context(), args)
|
||||
if err == nil || strings.Contains(err.Error(), "DATABASE_URL") {
|
||||
t.Errorf("run(%q) must report argument error, got %v", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedNeedsOnlyPostgres(t *testing.T) {
|
||||
db, err := testenv.StartPostgres(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
path := filepath.Join(t.TempDir(), "dsn")
|
||||
if err := os.WriteFile(path, []byte(db.DSN), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("DATABASE_URL_FILE", path)
|
||||
t.Setenv("BROKER_URL_FILE", "missing-broker")
|
||||
t.Setenv("S3_ENDPOINT", "invalid")
|
||||
t.Setenv("S3_SECRET_ACCESS_KEY_FILE", "")
|
||||
if err := run(t.Context(), []string{"failed"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -11,11 +11,23 @@ import (
|
||||
|
||||
var errorCodePattern = regexp.MustCompile("^[a-z][a-z0-9_]{0,63}$")
|
||||
|
||||
func (s *Store) Fail(ctx context.Context, c *Claim, code string, permanent bool, policy RetryPolicy) (bool, error) {
|
||||
type Failure struct {
|
||||
Code string
|
||||
Permanent bool
|
||||
}
|
||||
|
||||
type leaseState bool
|
||||
|
||||
const (
|
||||
activeLease leaseState = false
|
||||
expiredLease leaseState = true
|
||||
)
|
||||
|
||||
func (s *Store) Fail(ctx context.Context, c *Claim, failure Failure, policy RetryPolicy) (bool, error) {
|
||||
if err := policy.validate(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !errorCodePattern.MatchString(code) {
|
||||
if !errorCodePattern.MatchString(failure.Code) {
|
||||
return false, errors.New("invalid failure code")
|
||||
}
|
||||
var changed bool
|
||||
@@ -31,7 +43,7 @@ func (s *Store) Fail(ctx context.Context, c *Claim, code string, permanent bool,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = finishAttempt(ctx, tx, c.JobID, budget, code, permanent, false, policy)
|
||||
changed, err = finishAttempt(ctx, tx, c.JobID, budget, failure, activeLease, policy)
|
||||
return err
|
||||
})
|
||||
return changed && err == nil, err
|
||||
@@ -54,7 +66,7 @@ func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err = finishAttempt(ctx, tx, id, budget, "lease_expired", false, true, policy)
|
||||
changed, err = finishAttempt(ctx, tx, id, budget, Failure{Code: "lease_expired"}, expiredLease, policy)
|
||||
return err
|
||||
})
|
||||
return changed && err == nil, err
|
||||
@@ -62,13 +74,13 @@ func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error
|
||||
|
||||
// The caller holds the job row. Only the recovery path may advance an expired
|
||||
// attempt. A worker whose lease elapsed during its SQL call has lost ownership.
|
||||
func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, code string, permanent, expired bool, p RetryPolicy) (bool, error) {
|
||||
retry := !permanent && budget < p.MaxAttempts
|
||||
func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, failure Failure, lease leaseState, p RetryPolicy) (bool, error) {
|
||||
retry := !failure.Permanent && budget < p.MaxAttempts
|
||||
tag, err := tx.Exec(ctx, `UPDATE attachment_jobs
|
||||
SET state=CASE WHEN $3 THEN 'queued' ELSE 'failed' END,
|
||||
attempt=attempt+$3::boolean::int, budget_attempt=budget_attempt+$3::boolean::int,
|
||||
available_at=clock_timestamp()+$4::bigint*interval '1 microsecond',
|
||||
published_at=NULL, lease_token=NULL, lease_until=NULL, error_code=$2, updated_at=clock_timestamp()
|
||||
WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, code, retry, p.delay(budget).Microseconds(), expired)
|
||||
WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, failure.Code, retry, p.delay(budget).Microseconds(), bool(lease))
|
||||
return tag.RowsAffected() == 1, err
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ func TestPipelineTransitions(t *testing.T) {
|
||||
newJob(t)
|
||||
first := claim(t, publish(t))
|
||||
policy := RetryPolicy{MaxAttempts: 3, BaseDelay: 4 * time.Second, MaxDelay: 5 * time.Second}
|
||||
if ok, err := s.Fail(ctx, first, "storage_timeout", false, policy); !ok || err != nil {
|
||||
if ok, err := s.Fail(ctx, first, Failure{Code: "storage_timeout"}, policy); !ok || err != nil {
|
||||
t.Fatalf("failed %v %v", ok, err)
|
||||
}
|
||||
var seconds float64
|
||||
@@ -121,7 +121,7 @@ func TestPipelineTransitions(t *testing.T) {
|
||||
}
|
||||
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
|
||||
second := claim(t, publish(t))
|
||||
if ok, err := s.Fail(ctx, second, "storage_timeout", false, policy); !ok || err != nil {
|
||||
if ok, err := s.Fail(ctx, second, Failure{Code: "storage_timeout"}, policy); !ok || err != nil {
|
||||
t.Fatalf("second failure %v %v", ok, err)
|
||||
}
|
||||
if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil {
|
||||
@@ -177,7 +177,7 @@ func TestPipelineTransitions(t *testing.T) {
|
||||
if ok, err := s.Complete(ctx, old, variants(old)); err != nil || ok {
|
||||
t.Fatalf("stale commit %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, old, "source_missing", true, policy); err != nil || ok {
|
||||
if ok, err := s.Fail(ctx, old, Failure{Code: "source_missing", Permanent: true}, policy); err != nil || ok {
|
||||
t.Fatalf("stale failure %v %v", ok, err)
|
||||
}
|
||||
for _, dup := range []Job{current.Job, j, current.Job} {
|
||||
@@ -208,7 +208,7 @@ func TestPipelineTransitions(t *testing.T) {
|
||||
if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok {
|
||||
t.Fatalf("expired commit %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, first, "backend_unavailable", false, policy); err != nil || ok {
|
||||
if ok, err := s.Fail(ctx, first, Failure{Code: "backend_unavailable"}, policy); err != nil || ok {
|
||||
t.Fatalf("expired fail %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.RecoverOne(ctx, policy); err != nil || !ok {
|
||||
@@ -222,7 +222,7 @@ func TestPipelineTransitions(t *testing.T) {
|
||||
if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok {
|
||||
t.Fatalf("lost lease commit %v %v", ok, err)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, second, "backend_unavailable", false, policy); err != nil || !ok {
|
||||
if ok, err := s.Fail(ctx, second, Failure{Code: "backend_unavailable"}, policy); err != nil || !ok {
|
||||
t.Fatalf("second error %v %v", ok, err)
|
||||
}
|
||||
failed, err := s.Failed(ctx)
|
||||
@@ -247,12 +247,12 @@ func TestPipelineTransitions(t *testing.T) {
|
||||
if replay.Attempt != 3 || replay.JobID != first.JobID {
|
||||
t.Fatalf("replay identity %+v", replay)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, replay, "backend_unavailable", false, policy); !ok || err != nil {
|
||||
if ok, err := s.Fail(ctx, replay, Failure{Code: "backend_unavailable"}, policy); !ok || err != nil {
|
||||
t.Fatalf("fresh retry budget %v %v", ok, err)
|
||||
}
|
||||
exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`)
|
||||
last := claim(t, publish(t))
|
||||
if ok, err := s.Fail(ctx, last, "source_missing", true, policy); !ok || err != nil {
|
||||
if ok, err := s.Fail(ctx, last, Failure{Code: "source_missing", Permanent: true}, policy); !ok || err != nil {
|
||||
t.Fatalf("permanent fail %v %v", ok, err)
|
||||
}
|
||||
var n int
|
||||
@@ -289,7 +289,7 @@ func TestPipelineTransitions(t *testing.T) {
|
||||
if version != 1 || status != "ready" || resultCount != 0 {
|
||||
t.Fatalf("partial publication v%d status=%s rows=%d", version, status, resultCount)
|
||||
}
|
||||
if ok, err := s.Fail(ctx, next, "invalid_image", true, policy); err != nil || !ok {
|
||||
if ok, err := s.Fail(ctx, next, Failure{Code: "invalid_image", Permanent: true}, policy); err != nil || !ok {
|
||||
t.Fatalf("permanent error %v %v", ok, err)
|
||||
}
|
||||
failed, err := s.Failed(ctx)
|
||||
|
||||
@@ -12,8 +12,12 @@ type Jobs struct {
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func LoadBrokerURL(lookup Lookup) (string, error) {
|
||||
return secret(lookup, "BROKER_URL_FILE")
|
||||
}
|
||||
|
||||
func LoadJobs(lookup Lookup) (Jobs, error) {
|
||||
url, err := secret(lookup, "BROKER_URL_FILE")
|
||||
url, err := LoadBrokerURL(lookup)
|
||||
if err != nil {
|
||||
return Jobs{}, err
|
||||
}
|
||||
|
||||
@@ -25,46 +25,49 @@ func pause(ctx context.Context, d time.Duration) {
|
||||
}
|
||||
|
||||
func Dispatch(ctx context.Context, log *slog.Logger, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) {
|
||||
var p *broker.Publisher
|
||||
defer func() {
|
||||
if p != nil {
|
||||
p.Close()
|
||||
}
|
||||
}()
|
||||
for ctx.Err() == nil {
|
||||
_, err := store.RecoverOne(ctx, retryPolicy(cfg))
|
||||
if err == nil && p == nil {
|
||||
p, err = broker.NewPublisher(ctx, cfg.URL)
|
||||
}
|
||||
if err == nil {
|
||||
var sent bool
|
||||
sent, err = store.PublishOne(ctx, func(ctx context.Context, j pipeline.Job) error {
|
||||
body, err := json.Marshal(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.Publish(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
if barrier != nil {
|
||||
return barrier(ctx, j, "published")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
if !sent {
|
||||
pause(ctx, 250*time.Millisecond)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if p != nil {
|
||||
p.Close()
|
||||
p = nil
|
||||
}
|
||||
err := dispatchSession(ctx, store, cfg, barrier)
|
||||
if ctx.Err() == nil {
|
||||
log.Warn("attachment dispatcher will retry", "error", err)
|
||||
pause(ctx, time.Second)
|
||||
}
|
||||
pause(ctx, time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func dispatchSession(ctx context.Context, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) error {
|
||||
var publisher *broker.Publisher
|
||||
publish := func(ctx context.Context, j pipeline.Job) error {
|
||||
body, err := json.Marshal(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := publisher.Publish(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
if barrier != nil {
|
||||
return barrier(ctx, j, "published")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
if _, err := store.RecoverOne(ctx, retryPolicy(cfg)); err != nil {
|
||||
return err
|
||||
}
|
||||
if publisher == nil {
|
||||
var err error
|
||||
publisher, err = broker.NewPublisher(ctx, cfg.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer publisher.Close()
|
||||
}
|
||||
sent, err := store.PublishOne(ctx, publish)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sent {
|
||||
pause(ctx, 250*time.Millisecond)
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
@@ -6,11 +6,17 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/broker"
|
||||
"messenger/internal/imaging"
|
||||
@@ -95,23 +101,12 @@ func (w *Worker) process(ctx context.Context, j pipeline.Job) error {
|
||||
}
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout)
|
||||
defer cancel()
|
||||
variants, err := w.writeObjects(jobCtx, c)
|
||||
if err != nil {
|
||||
variants, failure := w.writeObjects(jobCtx, c)
|
||||
if failure != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
code := "processing_failed"
|
||||
permanent := false
|
||||
switch {
|
||||
case errors.Is(err, imaging.ErrInvalidImage):
|
||||
code, permanent = "invalid_image", true
|
||||
case errors.Is(err, errSourceMismatch):
|
||||
code, permanent = "source_mismatch", true
|
||||
case errors.Is(err, attachment.ErrObjectMissing):
|
||||
code = "source_missing"
|
||||
}
|
||||
w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code)
|
||||
_, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config))
|
||||
_, err = w.Store.Fail(ctx, c, *failure, retryPolicy(w.Config))
|
||||
return err
|
||||
}
|
||||
if err := w.checkpoint(ctx, j, "objects"); err != nil {
|
||||
@@ -125,7 +120,7 @@ func (w *Worker) process(ctx context.Context, j pipeline.Job) error {
|
||||
}
|
||||
|
||||
func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) {
|
||||
if c.SizeBytes < 1 || c.SizeBytes > 10<<20 {
|
||||
if c.SizeBytes < 1 || c.SizeBytes > imaging.MaxUploadBytes {
|
||||
return nil, errSourceMismatch
|
||||
}
|
||||
r, err := blobs.Open(ctx, c.SourceKey)
|
||||
@@ -147,19 +142,60 @@ func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) {
|
||||
// Log only classified reasons. Storage and codec errors may contain URLs,
|
||||
// object keys, filenames or input-derived diagnostics.
|
||||
func (w *Worker) attemptFailure(c *pipeline.Claim, stage string, err error) *pipeline.Failure {
|
||||
failure := &pipeline.Failure{Code: "processing_failed"}
|
||||
reason := "operation_failed"
|
||||
var network net.Error
|
||||
var exit *exec.ExitError
|
||||
var storage minio.ErrorResponse
|
||||
switch {
|
||||
case errors.Is(err, imaging.ErrInvalidImage):
|
||||
failure.Code, failure.Permanent = "invalid_image", true
|
||||
reason = failure.Code
|
||||
case errors.Is(err, errSourceMismatch):
|
||||
failure.Code, failure.Permanent = "source_mismatch", true
|
||||
reason = failure.Code
|
||||
case errors.Is(err, attachment.ErrObjectMissing):
|
||||
failure.Code = "source_missing"
|
||||
reason = failure.Code
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
reason = "deadline_exceeded"
|
||||
case errors.Is(err, context.Canceled):
|
||||
reason = "canceled"
|
||||
case errors.As(err, &exit):
|
||||
reason = fmt.Sprintf("codec_exit_%d", exit.ExitCode())
|
||||
case errors.Is(err, exec.ErrNotFound):
|
||||
reason = "codec_not_found"
|
||||
case errors.As(err, &storage):
|
||||
reason = fmt.Sprintf("s3_http_%d", storage.StatusCode)
|
||||
case errors.As(err, &network):
|
||||
reason = "network_error"
|
||||
if network.Timeout() {
|
||||
reason = "network_timeout"
|
||||
}
|
||||
case errors.Is(err, os.ErrPermission):
|
||||
reason = "permission_denied"
|
||||
}
|
||||
w.Log.Warn("attachment attempt failed", "job_id", c.JobID, "attempt", c.Attempt,
|
||||
"error_code", failure.Code, "stage", stage, "reason", reason)
|
||||
return failure
|
||||
}
|
||||
|
||||
func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, *pipeline.Failure) {
|
||||
data, err := readOriginal(ctx, w.Blobs, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, w.attemptFailure(c, "read_original", err)
|
||||
}
|
||||
encoded, err := imaging.Render(ctx, data, c.PipelineVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, w.attemptFailure(c, "render", err)
|
||||
}
|
||||
key := attachment.BlobKey(c.Hash)
|
||||
if c.SourceKey != key {
|
||||
if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil {
|
||||
return nil, err
|
||||
return nil, w.attemptFailure(c, "write_original", err)
|
||||
}
|
||||
}
|
||||
var variants []pipeline.Variant
|
||||
@@ -168,9 +204,12 @@ func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipelin
|
||||
key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:])
|
||||
n := int64(len(v.Data))
|
||||
if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil {
|
||||
return nil, err
|
||||
return nil, w.attemptFailure(c, "write_variant", err)
|
||||
}
|
||||
variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]})
|
||||
variants = append(variants, pipeline.Variant{
|
||||
Name: v.Name, Key: key, MIMEType: v.MIMEType,
|
||||
Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:],
|
||||
})
|
||||
}
|
||||
return variants, nil
|
||||
}
|
||||
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -24,7 +26,7 @@ func (s stagedSource) Open(context.Context, string) (io.ReadCloser, error) {
|
||||
func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) {
|
||||
h := sha256.Sum256([]byte("accepted"))
|
||||
c := &pipeline.Claim{SizeBytes: 8, Hash: h[:], SourceKey: "staging/test"}
|
||||
for _, bad := range []string{"truncated", "altered!", "oversized bytes"} {
|
||||
for _, bad := range []string{"accept", "altered!", "oversized bytes"} {
|
||||
_, err := readOriginal(t.Context(), stagedSource{body: bad}, c)
|
||||
if !errors.Is(err, errSourceMismatch) {
|
||||
t.Fatalf("body %q: %v", bad, err)
|
||||
@@ -35,3 +37,28 @@ func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) {
|
||||
t.Fatalf("valid original: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type failingSource struct{ attachment.Blobs }
|
||||
|
||||
func (failingSource) Open(context.Context, string) (io.ReadCloser, error) {
|
||||
return nil, fmt.Errorf("credentials=private-source-data: %w", context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
func TestFailureLogReportsStageAndReason(t *testing.T) {
|
||||
var log strings.Builder
|
||||
w := Worker{Blobs: failingSource{}, Log: slog.New(slog.NewJSONHandler(&log, nil))}
|
||||
c := &pipeline.Claim{SizeBytes: 5, SourceKey: "sensitive-source-key"}
|
||||
_, failure := w.writeObjects(t.Context(), c)
|
||||
if failure == nil {
|
||||
t.Fatal("expected failure reading source")
|
||||
}
|
||||
got := log.String()
|
||||
for _, want := range []string{`"stage":"read_original"`, `"reason":"deadline_exceeded"`, `"error_code":"processing_failed"`, `"job_id":`, `"attempt":`} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("log missing %s: %s", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Contains(got, "private-source-data") || strings.Contains(got, "sensitive-source-key") {
|
||||
t.Errorf("leaked source details: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user