diff --git a/cmd/worker/main.go b/cmd/worker/main.go index a27cb0b2..9f264c52 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -5,11 +5,9 @@ import ( "context" "encoding/json" "errors" - "log/slog" "os" "os/signal" - "strconv" "strings" "syscall" @@ -37,61 +35,110 @@ func main() { } } +const usage = "use failed, replay , reprocess , or duplicate " + +type command struct { + name, actor, barrierDir string + id uuid.UUID + version, attempt int +} + +func parseCommand(args []string) (command, error) { + var c command + if len(args) == 0 { + return c, nil + } + if dir, ok := strings.CutPrefix(args[0], "--barriers="); ok && len(args) == 1 && dir != "" { + c.barrierDir = dir + return c, nil + } + c.name = args[0] + switch c.name { + case "--prepare-large-png", "failed": + if len(args) == 1 { + return c, nil + } + return c, errors.New(usage) + case "replay", "reprocess": + if len(args) != 3 { + return c, errors.New(usage) + } + case "duplicate": + if len(args) != 4 { + return c, errors.New(usage) + } + default: + return c, errors.New(usage) + } + var err error + c.id, err = uuid.Parse(args[1]) + if err != nil { + return c, errors.New("invalid attachment UUID") + } + if c.name == "replay" { + c.actor = strings.TrimSpace(args[2]) + if c.actor == "" || len(c.actor) > 128 { + return c, errors.New("replay requires an operator name of up to 128 bytes") + } + return c, nil + } + c.version, err = strconv.Atoi(args[2]) + if err != nil || c.version < 1 { + return c, errors.New("invalid pipeline version") + } + if c.name == "duplicate" { + c.attempt, err = strconv.Atoi(args[3]) + if err != nil || c.attempt < 1 { + return c, errors.New("invalid attempt number") + } + } + return c, nil +} + func run(ctx context.Context, args []string) error { - if len(args) == 1 && args[0] == "--prepare-large-png" { + cmd, err := parseCommand(args) + if err != nil { + return err + } + if cmd.name == "--prepare-large-png" { return imaging.PrepareLargePNG(os.Stdin) } + if cmd.name != "" { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 15*time.Second) + defer cancel() + } db, err := config.LoadDatabaseURL(os.LookupEnv) if err != nil { return err } - cfg, err := config.LoadJobs(os.LookupEnv) - if err != nil { - return err - } - s3, err := config.LoadS3(os.LookupEnv) - if err != nil { - return err - } - blobs, err := s3store.New(s3) - if err != nil { - return err - } pool, err := postgres.NewPool(ctx, db, 5) if err != nil { return err } defer pool.Close() store := pipeline.NewStore(pool) - w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()} - if len(args) == 1 && strings.HasPrefix(args[0], "--barriers=") { - dir := strings.TrimPrefix(args[0], "--barriers=") - if err := os.MkdirAll(dir, 0700); err != nil { - return err - } - w.Checkpoint = processing.FileBarrier(dir) - args = nil - } - if len(args) == 0 { - w.Run(ctx) - return nil - } - op, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - if len(args) == 1 && args[0] == "failed" { - jobs, err := store.Failed(op) + switch cmd.name { + case "failed": + jobs, err := store.Failed(ctx) if err != nil { return err } return json.NewEncoder(os.Stdout).Encode(jobs) + case "duplicate": + url, err := config.LoadBrokerURL(os.LookupEnv) + if err != nil { + return err + } + return duplicate(ctx, pool, url, cmd.id, cmd.version, cmd.attempt) } - const usage = "use failed, replay , reprocess , or duplicate " - if len(args) < 3 { - return errors.New(usage) - } - id, err := uuid.Parse(args[1]) + s3, err := config.LoadS3(os.LookupEnv) if err != nil { - return errors.New("invalid attachment UUID") + return err + } + blobs, err := s3store.New(s3) + if err != nil { + return err } exists := func(ctx context.Context, key string) (bool, error) { _, err := blobs.Stat(ctx, key) @@ -100,37 +147,28 @@ func run(ctx context.Context, args []string) error { } return err == nil, err } - switch args[0] { + switch cmd.name { case "replay": - if len(args) != 3 { - return errors.New(usage) - } - return store.Replay(op, id, args[2], exists) + return store.Replay(ctx, cmd.id, cmd.actor, exists) case "reprocess": - if len(args) != 3 { - return errors.New(usage) - } - v, err := strconv.Atoi(args[2]) - if err != nil { - return err - } - return store.Reprocess(op, id, v, exists) - case "duplicate": - if len(args) != 4 { - return errors.New(usage) - } - version, err := strconv.Atoi(args[2]) - if err != nil { - return err - } - attempt, err := strconv.Atoi(args[3]) - if err != nil { - return err - } - return duplicate(op, pool, cfg.URL, id, version, attempt) - default: - return errors.New(usage) + return store.Reprocess(ctx, cmd.id, cmd.version, exists) } + cfg, err := config.LoadJobs(os.LookupEnv) + if err != nil { + return err + } + w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()} + if cmd.barrierDir != "" { + if err := os.MkdirAll(cmd.barrierDir, 0700); err != nil { + return err + } + w.Checkpoint = processing.FileBarrier(cmd.barrierDir) + } + if err := imaging.CleanScratch(); err != nil { + return err + } + w.Run(ctx) + return nil } // duplicate sends a previously issued envelope through the actual broker. It diff --git a/cmd/worker/main_test.go b/cmd/worker/main_test.go new file mode 100644 index 00000000..72ff8ad7 --- /dev/null +++ b/cmd/worker/main_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "messenger/internal/testenv" +) + +func TestInvalidCommandBeforeConfig(t *testing.T) { + t.Setenv("DATABASE_URL_FILE", "") + for _, args := range [][]string{{"unknown"}, {"replay", "bad-id", "admin"}, {"duplicate", "00000000-0000-4000-8000-000000000001", "1", "invalid"}} { + err := run(t.Context(), args) + if err == nil || strings.Contains(err.Error(), "DATABASE_URL") { + t.Errorf("run(%q) must report argument error, got %v", args, err) + } + } +} + +func TestFailedNeedsOnlyPostgres(t *testing.T) { + db, err := testenv.StartPostgres(t.Context()) + if err != nil { + t.Fatal(err) + } + defer db.Close() + path := filepath.Join(t.TempDir(), "dsn") + if err := os.WriteFile(path, []byte(db.DSN), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("DATABASE_URL_FILE", path) + t.Setenv("BROKER_URL_FILE", "missing-broker") + t.Setenv("S3_ENDPOINT", "invalid") + t.Setenv("S3_SECRET_ACCESS_KEY_FILE", "") + if err := run(t.Context(), []string{"failed"}); err != nil { + t.Fatal(err) + } +} diff --git a/internal/pipeline/retry.go b/internal/pipeline/retry.go index 0a0fc074..196a4389 100644 --- a/internal/pipeline/retry.go +++ b/internal/pipeline/retry.go @@ -11,11 +11,23 @@ import ( var errorCodePattern = regexp.MustCompile("^[a-z][a-z0-9_]{0,63}$") -func (s *Store) Fail(ctx context.Context, c *Claim, code string, permanent bool, policy RetryPolicy) (bool, error) { +type Failure struct { + Code string + Permanent bool +} + +type leaseState bool + +const ( + activeLease leaseState = false + expiredLease leaseState = true +) + +func (s *Store) Fail(ctx context.Context, c *Claim, failure Failure, policy RetryPolicy) (bool, error) { if err := policy.validate(); err != nil { return false, err } - if !errorCodePattern.MatchString(code) { + if !errorCodePattern.MatchString(failure.Code) { return false, errors.New("invalid failure code") } var changed bool @@ -31,7 +43,7 @@ func (s *Store) Fail(ctx context.Context, c *Claim, code string, permanent bool, if err != nil { return err } - changed, err = finishAttempt(ctx, tx, c.JobID, budget, code, permanent, false, policy) + changed, err = finishAttempt(ctx, tx, c.JobID, budget, failure, activeLease, policy) return err }) return changed && err == nil, err @@ -54,7 +66,7 @@ func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error if err != nil { return err } - changed, err = finishAttempt(ctx, tx, id, budget, "lease_expired", false, true, policy) + changed, err = finishAttempt(ctx, tx, id, budget, Failure{Code: "lease_expired"}, expiredLease, policy) return err }) return changed && err == nil, err @@ -62,13 +74,13 @@ func (s *Store) RecoverOne(ctx context.Context, policy RetryPolicy) (bool, error // The caller holds the job row. Only the recovery path may advance an expired // attempt. A worker whose lease elapsed during its SQL call has lost ownership. -func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, code string, permanent, expired bool, p RetryPolicy) (bool, error) { - retry := !permanent && budget < p.MaxAttempts +func finishAttempt(ctx context.Context, tx pgx.Tx, id uuid.UUID, budget int, failure Failure, lease leaseState, p RetryPolicy) (bool, error) { + retry := !failure.Permanent && budget < p.MaxAttempts tag, err := tx.Exec(ctx, `UPDATE attachment_jobs SET state=CASE WHEN $3 THEN 'queued' ELSE 'failed' END, attempt=attempt+$3::boolean::int, budget_attempt=budget_attempt+$3::boolean::int, available_at=clock_timestamp()+$4::bigint*interval '1 microsecond', published_at=NULL, lease_token=NULL, lease_until=NULL, error_code=$2, updated_at=clock_timestamp() - WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, code, retry, p.delay(budget).Microseconds(), expired) + WHERE job_id=$1 AND (lease_until<=clock_timestamp())=$5`, id, failure.Code, retry, p.delay(budget).Microseconds(), bool(lease)) return tag.RowsAffected() == 1, err } diff --git a/internal/pipeline/store_test.go b/internal/pipeline/store_test.go index 1c81942d..234e1691 100644 --- a/internal/pipeline/store_test.go +++ b/internal/pipeline/store_test.go @@ -101,7 +101,7 @@ func TestPipelineTransitions(t *testing.T) { newJob(t) first := claim(t, publish(t)) policy := RetryPolicy{MaxAttempts: 3, BaseDelay: 4 * time.Second, MaxDelay: 5 * time.Second} - if ok, err := s.Fail(ctx, first, "storage_timeout", false, policy); !ok || err != nil { + if ok, err := s.Fail(ctx, first, Failure{Code: "storage_timeout"}, policy); !ok || err != nil { t.Fatalf("failed %v %v", ok, err) } var seconds float64 @@ -121,7 +121,7 @@ func TestPipelineTransitions(t *testing.T) { } exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) second := claim(t, publish(t)) - if ok, err := s.Fail(ctx, second, "storage_timeout", false, policy); !ok || err != nil { + if ok, err := s.Fail(ctx, second, Failure{Code: "storage_timeout"}, policy); !ok || err != nil { t.Fatalf("second failure %v %v", ok, err) } if err := db.Pool.QueryRow(ctx, `SELECT extract(epoch FROM available_at-updated_at)::float8 FROM attachment_jobs`).Scan(&seconds); err != nil { @@ -177,7 +177,7 @@ func TestPipelineTransitions(t *testing.T) { if ok, err := s.Complete(ctx, old, variants(old)); err != nil || ok { t.Fatalf("stale commit %v %v", ok, err) } - if ok, err := s.Fail(ctx, old, "source_missing", true, policy); err != nil || ok { + if ok, err := s.Fail(ctx, old, Failure{Code: "source_missing", Permanent: true}, policy); err != nil || ok { t.Fatalf("stale failure %v %v", ok, err) } for _, dup := range []Job{current.Job, j, current.Job} { @@ -208,7 +208,7 @@ func TestPipelineTransitions(t *testing.T) { if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok { t.Fatalf("expired commit %v %v", ok, err) } - if ok, err := s.Fail(ctx, first, "backend_unavailable", false, policy); err != nil || ok { + if ok, err := s.Fail(ctx, first, Failure{Code: "backend_unavailable"}, policy); err != nil || ok { t.Fatalf("expired fail %v %v", ok, err) } if ok, err := s.RecoverOne(ctx, policy); err != nil || !ok { @@ -222,7 +222,7 @@ func TestPipelineTransitions(t *testing.T) { if ok, err := s.Complete(ctx, first, variants(first)); err != nil || ok { t.Fatalf("lost lease commit %v %v", ok, err) } - if ok, err := s.Fail(ctx, second, "backend_unavailable", false, policy); err != nil || !ok { + if ok, err := s.Fail(ctx, second, Failure{Code: "backend_unavailable"}, policy); err != nil || !ok { t.Fatalf("second error %v %v", ok, err) } failed, err := s.Failed(ctx) @@ -247,12 +247,12 @@ func TestPipelineTransitions(t *testing.T) { if replay.Attempt != 3 || replay.JobID != first.JobID { t.Fatalf("replay identity %+v", replay) } - if ok, err := s.Fail(ctx, replay, "backend_unavailable", false, policy); !ok || err != nil { + if ok, err := s.Fail(ctx, replay, Failure{Code: "backend_unavailable"}, policy); !ok || err != nil { t.Fatalf("fresh retry budget %v %v", ok, err) } exec(t, `UPDATE attachment_jobs SET available_at=clock_timestamp()`) last := claim(t, publish(t)) - if ok, err := s.Fail(ctx, last, "source_missing", true, policy); !ok || err != nil { + if ok, err := s.Fail(ctx, last, Failure{Code: "source_missing", Permanent: true}, policy); !ok || err != nil { t.Fatalf("permanent fail %v %v", ok, err) } var n int @@ -289,7 +289,7 @@ func TestPipelineTransitions(t *testing.T) { if version != 1 || status != "ready" || resultCount != 0 { t.Fatalf("partial publication v%d status=%s rows=%d", version, status, resultCount) } - if ok, err := s.Fail(ctx, next, "invalid_image", true, policy); err != nil || !ok { + if ok, err := s.Fail(ctx, next, Failure{Code: "invalid_image", Permanent: true}, policy); err != nil || !ok { t.Fatalf("permanent error %v %v", ok, err) } failed, err := s.Failed(ctx) diff --git a/internal/platform/config/jobs.go b/internal/platform/config/jobs.go index 8b497f04..1e550e49 100644 --- a/internal/platform/config/jobs.go +++ b/internal/platform/config/jobs.go @@ -12,8 +12,12 @@ type Jobs struct { Timeout time.Duration } +func LoadBrokerURL(lookup Lookup) (string, error) { + return secret(lookup, "BROKER_URL_FILE") +} + func LoadJobs(lookup Lookup) (Jobs, error) { - url, err := secret(lookup, "BROKER_URL_FILE") + url, err := LoadBrokerURL(lookup) if err != nil { return Jobs{}, err } diff --git a/internal/processing/dispatcher.go b/internal/processing/dispatcher.go index 376655ec..725a3cc5 100644 --- a/internal/processing/dispatcher.go +++ b/internal/processing/dispatcher.go @@ -25,46 +25,49 @@ func pause(ctx context.Context, d time.Duration) { } func Dispatch(ctx context.Context, log *slog.Logger, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) { - var p *broker.Publisher - defer func() { - if p != nil { - p.Close() - } - }() for ctx.Err() == nil { - _, err := store.RecoverOne(ctx, retryPolicy(cfg)) - if err == nil && p == nil { - p, err = broker.NewPublisher(ctx, cfg.URL) - } - if err == nil { - var sent bool - sent, err = store.PublishOne(ctx, func(ctx context.Context, j pipeline.Job) error { - body, err := json.Marshal(j) - if err != nil { - return err - } - if err := p.Publish(ctx, body); err != nil { - return err - } - if barrier != nil { - return barrier(ctx, j, "published") - } - return nil - }) - if err == nil { - if !sent { - pause(ctx, 250*time.Millisecond) - } - continue - } - } - if p != nil { - p.Close() - p = nil - } + err := dispatchSession(ctx, store, cfg, barrier) if ctx.Err() == nil { log.Warn("attachment dispatcher will retry", "error", err) + pause(ctx, time.Second) } - pause(ctx, time.Second) } } + +func dispatchSession(ctx context.Context, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) error { + var publisher *broker.Publisher + publish := func(ctx context.Context, j pipeline.Job) error { + body, err := json.Marshal(j) + if err != nil { + return err + } + if err := publisher.Publish(ctx, body); err != nil { + return err + } + if barrier != nil { + return barrier(ctx, j, "published") + } + return nil + } + for ctx.Err() == nil { + if _, err := store.RecoverOne(ctx, retryPolicy(cfg)); err != nil { + return err + } + if publisher == nil { + var err error + publisher, err = broker.NewPublisher(ctx, cfg.URL) + if err != nil { + return err + } + defer publisher.Close() + } + sent, err := store.PublishOne(ctx, publish) + if err != nil { + return err + } + if !sent { + pause(ctx, 250*time.Millisecond) + } + } + return ctx.Err() +} diff --git a/internal/processing/worker.go b/internal/processing/worker.go index 56544598..1574bc0b 100644 --- a/internal/processing/worker.go +++ b/internal/processing/worker.go @@ -6,11 +6,17 @@ import ( "crypto/sha256" "encoding/json" "errors" + "fmt" "io" "log/slog" + "net" + "os" + "os/exec" "sync" "time" + "github.com/minio/minio-go/v7" + "messenger/internal/attachment" "messenger/internal/broker" "messenger/internal/imaging" @@ -95,23 +101,12 @@ func (w *Worker) process(ctx context.Context, j pipeline.Job) error { } jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout) defer cancel() - variants, err := w.writeObjects(jobCtx, c) - if err != nil { + variants, failure := w.writeObjects(jobCtx, c) + if failure != nil { if ctx.Err() != nil { return ctx.Err() } - code := "processing_failed" - permanent := false - switch { - case errors.Is(err, imaging.ErrInvalidImage): - code, permanent = "invalid_image", true - case errors.Is(err, errSourceMismatch): - code, permanent = "source_mismatch", true - case errors.Is(err, attachment.ErrObjectMissing): - code = "source_missing" - } - w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code) - _, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config)) + _, err = w.Store.Fail(ctx, c, *failure, retryPolicy(w.Config)) return err } if err := w.checkpoint(ctx, j, "objects"); err != nil { @@ -125,7 +120,7 @@ func (w *Worker) process(ctx context.Context, j pipeline.Job) error { } func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) { - if c.SizeBytes < 1 || c.SizeBytes > 10<<20 { + if c.SizeBytes < 1 || c.SizeBytes > imaging.MaxUploadBytes { return nil, errSourceMismatch } r, err := blobs.Open(ctx, c.SourceKey) @@ -147,19 +142,60 @@ func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim return data, nil } -func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) { +// Log only classified reasons. Storage and codec errors may contain URLs, +// object keys, filenames or input-derived diagnostics. +func (w *Worker) attemptFailure(c *pipeline.Claim, stage string, err error) *pipeline.Failure { + failure := &pipeline.Failure{Code: "processing_failed"} + reason := "operation_failed" + var network net.Error + var exit *exec.ExitError + var storage minio.ErrorResponse + switch { + case errors.Is(err, imaging.ErrInvalidImage): + failure.Code, failure.Permanent = "invalid_image", true + reason = failure.Code + case errors.Is(err, errSourceMismatch): + failure.Code, failure.Permanent = "source_mismatch", true + reason = failure.Code + case errors.Is(err, attachment.ErrObjectMissing): + failure.Code = "source_missing" + reason = failure.Code + case errors.Is(err, context.DeadlineExceeded): + reason = "deadline_exceeded" + case errors.Is(err, context.Canceled): + reason = "canceled" + case errors.As(err, &exit): + reason = fmt.Sprintf("codec_exit_%d", exit.ExitCode()) + case errors.Is(err, exec.ErrNotFound): + reason = "codec_not_found" + case errors.As(err, &storage): + reason = fmt.Sprintf("s3_http_%d", storage.StatusCode) + case errors.As(err, &network): + reason = "network_error" + if network.Timeout() { + reason = "network_timeout" + } + case errors.Is(err, os.ErrPermission): + reason = "permission_denied" + } + w.Log.Warn("attachment attempt failed", "job_id", c.JobID, "attempt", c.Attempt, + "error_code", failure.Code, "stage", stage, "reason", reason) + return failure +} + +func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, *pipeline.Failure) { data, err := readOriginal(ctx, w.Blobs, c) if err != nil { - return nil, err + return nil, w.attemptFailure(c, "read_original", err) } encoded, err := imaging.Render(ctx, data, c.PipelineVersion) if err != nil { - return nil, err + return nil, w.attemptFailure(c, "render", err) } key := attachment.BlobKey(c.Hash) if c.SourceKey != key { if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil { - return nil, err + return nil, w.attemptFailure(c, "write_original", err) } } var variants []pipeline.Variant @@ -168,9 +204,12 @@ func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipelin key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:]) n := int64(len(v.Data)) if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil { - return nil, err + return nil, w.attemptFailure(c, "write_variant", err) } - variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]}) + variants = append(variants, pipeline.Variant{ + Name: v.Name, Key: key, MIMEType: v.MIMEType, + Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:], + }) } return variants, nil } diff --git a/internal/processing/worker_test.go b/internal/processing/worker_test.go index b021d7aa..45d2bce4 100644 --- a/internal/processing/worker_test.go +++ b/internal/processing/worker_test.go @@ -4,7 +4,9 @@ import ( "context" "crypto/sha256" "errors" + "fmt" "io" + "log/slog" "strings" "testing" @@ -24,7 +26,7 @@ func (s stagedSource) Open(context.Context, string) (io.ReadCloser, error) { func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) { h := sha256.Sum256([]byte("accepted")) c := &pipeline.Claim{SizeBytes: 8, Hash: h[:], SourceKey: "staging/test"} - for _, bad := range []string{"truncated", "altered!", "oversized bytes"} { + for _, bad := range []string{"accept", "altered!", "oversized bytes"} { _, err := readOriginal(t.Context(), stagedSource{body: bad}, c) if !errors.Is(err, errSourceMismatch) { t.Fatalf("body %q: %v", bad, err) @@ -35,3 +37,28 @@ func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) { t.Fatalf("valid original: %v", err) } } + +type failingSource struct{ attachment.Blobs } + +func (failingSource) Open(context.Context, string) (io.ReadCloser, error) { + return nil, fmt.Errorf("credentials=private-source-data: %w", context.DeadlineExceeded) +} + +func TestFailureLogReportsStageAndReason(t *testing.T) { + var log strings.Builder + w := Worker{Blobs: failingSource{}, Log: slog.New(slog.NewJSONHandler(&log, nil))} + c := &pipeline.Claim{SizeBytes: 5, SourceKey: "sensitive-source-key"} + _, failure := w.writeObjects(t.Context(), c) + if failure == nil { + t.Fatal("expected failure reading source") + } + got := log.String() + for _, want := range []string{`"stage":"read_original"`, `"reason":"deadline_exceeded"`, `"error_code":"processing_failed"`, `"job_id":`, `"attempt":`} { + if !strings.Contains(got, want) { + t.Errorf("log missing %s: %s", want, got) + } + } + if strings.Contains(got, "private-source-data") || strings.Contains(got, "sensitive-source-key") { + t.Errorf("leaked source details: %s", got) + } +}