package processing import ( "bytes" "context" "crypto/sha256" "encoding/json" "errors" "io" "log/slog" "sync" "time" "messenger/internal/attachment" "messenger/internal/broker" "messenger/internal/imaging" "messenger/internal/pipeline" "messenger/internal/platform/config" ) var errSourceMismatch = errors.New("original differs from accepted metadata") type Worker struct { Store *pipeline.Store Blobs attachment.Blobs Config config.Jobs Log *slog.Logger // An experiment may pause an individual attempt at an exact durability boundary. Checkpoint func(context.Context, pipeline.Job, string) error } func (w *Worker) Run(ctx context.Context) { var wg sync.WaitGroup for range w.Config.Concurrency { wg.Go(func() { for ctx.Err() == nil { err := w.consume(ctx) if ctx.Err() == nil { w.Log.Warn("consumer reconnecting", "error", err) pause(ctx, time.Second) } } }) } wg.Wait() } func (w *Worker) consume(ctx context.Context) error { c, err := broker.NewConsumer(ctx, w.Config.URL) if err != nil { return err } defer c.Close() for d := range c.Deliveries { var j pipeline.Job dec := json.NewDecoder(bytes.NewReader(d.Body)) dec.DisallowUnknownFields() if err := dec.Decode(&j); err != nil || !j.Valid() || dec.Decode(new(any)) != io.EOF { w.Log.Warn("discarding invalid attachment envelope") if err := d.Reject(false); err != nil { return err } continue } if err := w.process(ctx, j); err != nil { return err } if err := d.Ack(false); err != nil { return err } if err := w.checkpoint(ctx, j, "acked"); err != nil { return err } } return errors.New("broker delivery stream closed") } func (w *Worker) checkpoint(ctx context.Context, j pipeline.Job, phase string) error { if w.Checkpoint != nil { return w.Checkpoint(ctx, j, phase) } return nil } func (w *Worker) process(ctx context.Context, j pipeline.Job) error { c, err := w.Store.Claim(ctx, j, w.Config.Timeout+5*time.Second) if err != nil { return err } if c == nil { return w.checkpoint(ctx, j, "duplicate") } if err := w.checkpoint(ctx, j, "claimed"); err != nil { return err } jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout) defer cancel() variants, err := w.writeObjects(jobCtx, c) if err != nil { if ctx.Err() != nil { return ctx.Err() } code := "processing_failed" permanent := false switch { case errors.Is(err, imaging.ErrInvalidImage): code, permanent = "invalid_image", true case errors.Is(err, errSourceMismatch): code, permanent = "source_mismatch", true case errors.Is(err, attachment.ErrObjectMissing): code = "source_missing" } w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code) _, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config)) return err } if err := w.checkpoint(ctx, j, "objects"); err != nil { return err } saved, err := w.Store.Complete(ctx, c, variants) if err != nil || !saved { return err } return w.checkpoint(ctx, j, "committed") } func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) { if c.SizeBytes < 1 || c.SizeBytes > 10<<20 { return nil, errSourceMismatch } r, err := blobs.Open(ctx, c.SourceKey) if err != nil { return nil, err } data, err := io.ReadAll(io.LimitReader(r, c.SizeBytes+1)) closeErr := r.Close() if err != nil { return nil, err } if closeErr != nil { return nil, closeErr } h := sha256.Sum256(data) if int64(len(data)) != c.SizeBytes || !bytes.Equal(h[:], c.Hash) { return nil, errSourceMismatch } return data, nil } func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) { data, err := readOriginal(ctx, w.Blobs, c) if err != nil { return nil, err } encoded, err := imaging.Render(ctx, data, c.PipelineVersion) if err != nil { return nil, err } key := attachment.BlobKey(c.Hash) if c.SourceKey != key { if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil { return nil, err } } var variants []pipeline.Variant for _, v := range encoded { hash := sha256.Sum256(v.Data) key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:]) n := int64(len(v.Data)) if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil { return nil, err } variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]}) } return variants, nil }