Files
mtusi2026lab5/internal/processing/worker.go
T

177 lines
4.5 KiB
Go

package processing
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"io"
"log/slog"
"sync"
"time"
"messenger/internal/attachment"
"messenger/internal/broker"
"messenger/internal/imaging"
"messenger/internal/pipeline"
"messenger/internal/platform/config"
)
var errSourceMismatch = errors.New("original differs from accepted metadata")
type Worker struct {
Store *pipeline.Store
Blobs attachment.Blobs
Config config.Jobs
Log *slog.Logger
// An experiment may pause an individual attempt at an exact durability boundary.
Checkpoint func(context.Context, pipeline.Job, string) error
}
func (w *Worker) Run(ctx context.Context) {
var wg sync.WaitGroup
for range w.Config.Concurrency {
wg.Go(func() {
for ctx.Err() == nil {
err := w.consume(ctx)
if ctx.Err() == nil {
w.Log.Warn("consumer reconnecting", "error", err)
pause(ctx, time.Second)
}
}
})
}
wg.Wait()
}
func (w *Worker) consume(ctx context.Context) error {
c, err := broker.NewConsumer(ctx, w.Config.URL)
if err != nil {
return err
}
defer c.Close()
for d := range c.Deliveries {
var j pipeline.Job
dec := json.NewDecoder(bytes.NewReader(d.Body))
dec.DisallowUnknownFields()
if err := dec.Decode(&j); err != nil || !j.Valid() || dec.Decode(new(any)) != io.EOF {
w.Log.Warn("discarding invalid attachment envelope")
if err := d.Reject(false); err != nil {
return err
}
continue
}
if err := w.process(ctx, j); err != nil {
return err
}
if err := d.Ack(false); err != nil {
return err
}
if err := w.checkpoint(ctx, j, "acked"); err != nil {
return err
}
}
return errors.New("broker delivery stream closed")
}
func (w *Worker) checkpoint(ctx context.Context, j pipeline.Job, phase string) error {
if w.Checkpoint != nil {
return w.Checkpoint(ctx, j, phase)
}
return nil
}
func (w *Worker) process(ctx context.Context, j pipeline.Job) error {
c, err := w.Store.Claim(ctx, j, w.Config.Timeout+5*time.Second)
if err != nil {
return err
}
if c == nil {
return w.checkpoint(ctx, j, "duplicate")
}
if err := w.checkpoint(ctx, j, "claimed"); err != nil {
return err
}
jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout)
defer cancel()
variants, err := w.writeObjects(jobCtx, c)
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
code := "processing_failed"
permanent := false
switch {
case errors.Is(err, imaging.ErrInvalidImage):
code, permanent = "invalid_image", true
case errors.Is(err, errSourceMismatch):
code, permanent = "source_mismatch", true
case errors.Is(err, attachment.ErrObjectMissing):
code = "source_missing"
}
w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code)
_, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config))
return err
}
if err := w.checkpoint(ctx, j, "objects"); err != nil {
return err
}
saved, err := w.Store.Complete(ctx, c, variants)
if err != nil || !saved {
return err
}
return w.checkpoint(ctx, j, "committed")
}
func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) {
if c.SizeBytes < 1 || c.SizeBytes > 10<<20 {
return nil, errSourceMismatch
}
r, err := blobs.Open(ctx, c.SourceKey)
if err != nil {
return nil, err
}
data, err := io.ReadAll(io.LimitReader(r, c.SizeBytes+1))
closeErr := r.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
h := sha256.Sum256(data)
if int64(len(data)) != c.SizeBytes || !bytes.Equal(h[:], c.Hash) {
return nil, errSourceMismatch
}
return data, nil
}
func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) {
data, err := readOriginal(ctx, w.Blobs, c)
if err != nil {
return nil, err
}
encoded, err := imaging.Render(ctx, data, c.PipelineVersion)
if err != nil {
return nil, err
}
key := attachment.BlobKey(c.Hash)
if c.SourceKey != key {
if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil {
return nil, err
}
}
var variants []pipeline.Variant
for _, v := range encoded {
hash := sha256.Sum256(v.Data)
key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:])
n := int64(len(v.Data))
if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil {
return nil, err
}
variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]})
}
return variants, nil
}