Разделите настройки команд worker и сохраняйте причину сбоя

This commit is contained in:
2026-11-20 10:54:34 +03:00
parent 42a9bf2071
commit 8f90c78b98
8 changed files with 303 additions and 141 deletions
+40 -37
View File
@@ -25,46 +25,49 @@ func pause(ctx context.Context, d time.Duration) {
}
func Dispatch(ctx context.Context, log *slog.Logger, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) {
var p *broker.Publisher
defer func() {
if p != nil {
p.Close()
}
}()
for ctx.Err() == nil {
_, err := store.RecoverOne(ctx, retryPolicy(cfg))
if err == nil && p == nil {
p, err = broker.NewPublisher(ctx, cfg.URL)
}
if err == nil {
var sent bool
sent, err = store.PublishOne(ctx, func(ctx context.Context, j pipeline.Job) error {
body, err := json.Marshal(j)
if err != nil {
return err
}
if err := p.Publish(ctx, body); err != nil {
return err
}
if barrier != nil {
return barrier(ctx, j, "published")
}
return nil
})
if err == nil {
if !sent {
pause(ctx, 250*time.Millisecond)
}
continue
}
}
if p != nil {
p.Close()
p = nil
}
err := dispatchSession(ctx, store, cfg, barrier)
if ctx.Err() == nil {
log.Warn("attachment dispatcher will retry", "error", err)
pause(ctx, time.Second)
}
pause(ctx, time.Second)
}
}
func dispatchSession(ctx context.Context, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) error {
var publisher *broker.Publisher
publish := func(ctx context.Context, j pipeline.Job) error {
body, err := json.Marshal(j)
if err != nil {
return err
}
if err := publisher.Publish(ctx, body); err != nil {
return err
}
if barrier != nil {
return barrier(ctx, j, "published")
}
return nil
}
for ctx.Err() == nil {
if _, err := store.RecoverOne(ctx, retryPolicy(cfg)); err != nil {
return err
}
if publisher == nil {
var err error
publisher, err = broker.NewPublisher(ctx, cfg.URL)
if err != nil {
return err
}
defer publisher.Close()
}
sent, err := store.PublishOne(ctx, publish)
if err != nil {
return err
}
if !sent {
pause(ctx, 250*time.Millisecond)
}
}
return ctx.Err()
}
+60 -21
View File
@@ -6,11 +6,17 @@ import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net"
"os"
"os/exec"
"sync"
"time"
"github.com/minio/minio-go/v7"
"messenger/internal/attachment"
"messenger/internal/broker"
"messenger/internal/imaging"
@@ -95,23 +101,12 @@ func (w *Worker) process(ctx context.Context, j pipeline.Job) error {
}
jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout)
defer cancel()
variants, err := w.writeObjects(jobCtx, c)
if err != nil {
variants, failure := w.writeObjects(jobCtx, c)
if failure != nil {
if ctx.Err() != nil {
return ctx.Err()
}
code := "processing_failed"
permanent := false
switch {
case errors.Is(err, imaging.ErrInvalidImage):
code, permanent = "invalid_image", true
case errors.Is(err, errSourceMismatch):
code, permanent = "source_mismatch", true
case errors.Is(err, attachment.ErrObjectMissing):
code = "source_missing"
}
w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code)
_, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config))
_, err = w.Store.Fail(ctx, c, *failure, retryPolicy(w.Config))
return err
}
if err := w.checkpoint(ctx, j, "objects"); err != nil {
@@ -125,7 +120,7 @@ func (w *Worker) process(ctx context.Context, j pipeline.Job) error {
}
func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) {
if c.SizeBytes < 1 || c.SizeBytes > 10<<20 {
if c.SizeBytes < 1 || c.SizeBytes > imaging.MaxUploadBytes {
return nil, errSourceMismatch
}
r, err := blobs.Open(ctx, c.SourceKey)
@@ -147,19 +142,60 @@ func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim
return data, nil
}
func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) {
// Log only classified reasons. Storage and codec errors may contain URLs,
// object keys, filenames or input-derived diagnostics.
func (w *Worker) attemptFailure(c *pipeline.Claim, stage string, err error) *pipeline.Failure {
failure := &pipeline.Failure{Code: "processing_failed"}
reason := "operation_failed"
var network net.Error
var exit *exec.ExitError
var storage minio.ErrorResponse
switch {
case errors.Is(err, imaging.ErrInvalidImage):
failure.Code, failure.Permanent = "invalid_image", true
reason = failure.Code
case errors.Is(err, errSourceMismatch):
failure.Code, failure.Permanent = "source_mismatch", true
reason = failure.Code
case errors.Is(err, attachment.ErrObjectMissing):
failure.Code = "source_missing"
reason = failure.Code
case errors.Is(err, context.DeadlineExceeded):
reason = "deadline_exceeded"
case errors.Is(err, context.Canceled):
reason = "canceled"
case errors.As(err, &exit):
reason = fmt.Sprintf("codec_exit_%d", exit.ExitCode())
case errors.Is(err, exec.ErrNotFound):
reason = "codec_not_found"
case errors.As(err, &storage):
reason = fmt.Sprintf("s3_http_%d", storage.StatusCode)
case errors.As(err, &network):
reason = "network_error"
if network.Timeout() {
reason = "network_timeout"
}
case errors.Is(err, os.ErrPermission):
reason = "permission_denied"
}
w.Log.Warn("attachment attempt failed", "job_id", c.JobID, "attempt", c.Attempt,
"error_code", failure.Code, "stage", stage, "reason", reason)
return failure
}
func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, *pipeline.Failure) {
data, err := readOriginal(ctx, w.Blobs, c)
if err != nil {
return nil, err
return nil, w.attemptFailure(c, "read_original", err)
}
encoded, err := imaging.Render(ctx, data, c.PipelineVersion)
if err != nil {
return nil, err
return nil, w.attemptFailure(c, "render", err)
}
key := attachment.BlobKey(c.Hash)
if c.SourceKey != key {
if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil {
return nil, err
return nil, w.attemptFailure(c, "write_original", err)
}
}
var variants []pipeline.Variant
@@ -168,9 +204,12 @@ func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipelin
key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:])
n := int64(len(v.Data))
if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil {
return nil, err
return nil, w.attemptFailure(c, "write_variant", err)
}
variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]})
variants = append(variants, pipeline.Variant{
Name: v.Name, Key: key, MIMEType: v.MIMEType,
Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:],
})
}
return variants, nil
}
+28 -1
View File
@@ -4,7 +4,9 @@ import (
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"log/slog"
"strings"
"testing"
@@ -24,7 +26,7 @@ func (s stagedSource) Open(context.Context, string) (io.ReadCloser, error) {
func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) {
h := sha256.Sum256([]byte("accepted"))
c := &pipeline.Claim{SizeBytes: 8, Hash: h[:], SourceKey: "staging/test"}
for _, bad := range []string{"truncated", "altered!", "oversized bytes"} {
for _, bad := range []string{"accept", "altered!", "oversized bytes"} {
_, err := readOriginal(t.Context(), stagedSource{body: bad}, c)
if !errors.Is(err, errSourceMismatch) {
t.Fatalf("body %q: %v", bad, err)
@@ -35,3 +37,28 @@ func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) {
t.Fatalf("valid original: %v", err)
}
}
type failingSource struct{ attachment.Blobs }
func (failingSource) Open(context.Context, string) (io.ReadCloser, error) {
return nil, fmt.Errorf("credentials=private-source-data: %w", context.DeadlineExceeded)
}
func TestFailureLogReportsStageAndReason(t *testing.T) {
var log strings.Builder
w := Worker{Blobs: failingSource{}, Log: slog.New(slog.NewJSONHandler(&log, nil))}
c := &pipeline.Claim{SizeBytes: 5, SourceKey: "sensitive-source-key"}
_, failure := w.writeObjects(t.Context(), c)
if failure == nil {
t.Fatal("expected failure reading source")
}
got := log.String()
for _, want := range []string{`"stage":"read_original"`, `"reason":"deadline_exceeded"`, `"error_code":"processing_failed"`, `"job_id":`, `"attempt":`} {
if !strings.Contains(got, want) {
t.Errorf("log missing %s: %s", want, got)
}
}
if strings.Contains(got, "private-source-data") || strings.Contains(got, "sensitive-source-key") {
t.Errorf("leaked source details: %s", got)
}
}