Files
mtusi2026lab6/internal/messenger/attachment/worker/worker.go
T

544 lines
18 KiB
Go

package worker
import (
"bytes"
"context"
"errors"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"log/slog"
"math"
"path"
"strings"
"sync"
"time"
"golang.org/x/sync/semaphore"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
)
type Repository interface {
AttachmentJob(context.Context, string) (model.AttachmentJob, error)
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error)
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
}
type Broker interface {
Read(context.Context) ([]redisstream.Delivery, error)
Reclaim(context.Context, string) ([]redisstream.Delivery, string, error)
}
type ObjectStore interface {
Get(context.Context, string) (io.ReadCloser, error)
Put(context.Context, string, string, []byte) error
Delete(context.Context, string) error
}
type ImageProcessor interface {
Process(context.Context, []byte, string) (transform.Result, error)
}
type Metrics interface {
ObserveStarted(context.Context)
ObserveFinished(context.Context, string, time.Duration)
ObserveTimeToReady(context.Context, time.Duration)
ObserveRetry(context.Context)
ObserveReclaimed(context.Context, int)
}
type Config struct {
WorkerID string
Concurrency int
LeaseDuration time.Duration
HeartbeatInterval time.Duration
ProcessingTimeout time.Duration
ReclaimInterval time.Duration
MaxAttempts int
RetryBase time.Duration
RetryMax time.Duration
MaxBytes int64
MaxPixels int64
MemoryBytes int64
}
type FaultPoint string
const (
FaultNone FaultPoint = ""
FaultBeforeSideEffect FaultPoint = "before_side_effect"
FaultAfterSideEffect FaultPoint = "after_side_effect"
FaultAfterCommit FaultPoint = "after_commit"
)
var ErrInjectedFailure = errors.New("injected worker failure")
type Worker struct {
logger *slog.Logger
repository Repository
broker Broker
objects ObjectStore
processor ImageProcessor
metrics Metrics
config Config
memory *semaphore.Weighted
now func() time.Time
}
func New(
logger *slog.Logger,
repository Repository,
broker Broker,
objects ObjectStore,
processor ImageProcessor,
metrics Metrics,
config Config,
) (*Worker, error) {
if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil {
return nil, errors.New("worker dependencies are required")
}
if err := validateConfig(config); err != nil {
return nil, err
}
if metrics == nil {
metrics = nopMetrics{}
}
return &Worker{
logger: logger, repository: repository, broker: broker, objects: objects,
processor: processor, metrics: metrics, config: config,
memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now,
}, nil
}
func validateConfig(config Config) error {
if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 ||
config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration ||
config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration ||
config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 ||
config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 {
return errors.New("invalid worker configuration")
}
const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128
if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 ||
config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes {
return errors.New("worker memory must fit one maximum image")
}
return nil
}
func (w *Worker) Run(ctx context.Context) error {
lastReclaim := time.Time{}
reclaimCursor := "0-0"
for ctx.Err() == nil {
deliveries, err := w.broker.Read(ctx)
if err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
w.processBatch(ctx, deliveries)
if w.now().Sub(lastReclaim) < w.config.ReclaimInterval {
continue
}
recovery, err := w.repository.RecoverExpiredAttachmentJobs(
ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase),
)
if err != nil {
return fmt.Errorf("recover expired attachment jobs: %w", err)
}
if recovery.Requeued > 0 || recovery.Failed > 0 {
w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed)
}
deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor)
if err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
w.metrics.ObserveReclaimed(ctx, len(deliveries))
w.processBatch(ctx, deliveries)
reclaimCursor = next
if reclaimCursor == "" {
reclaimCursor = "0-0"
}
lastReclaim = w.now()
}
return nil
}
func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) {
limit := make(chan struct{}, w.config.Concurrency)
var group sync.WaitGroup
for _, delivery := range deliveries {
if ctx.Err() != nil {
break
}
limit <- struct{}{}
group.Add(1)
go func(delivery redisstream.Delivery) {
defer group.Done()
defer func() { <-limit }()
_ = w.ProcessDelivery(ctx, delivery, FaultNone)
}(delivery)
}
group.Wait()
}
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) {
started := w.now()
result := "transient_error"
if delivery.DecodeError != nil {
w.metrics.ObserveStarted(ctx)
defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }()
result = "invalid_job"
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
}
envelope := delivery.Envelope
ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{
Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate,
})
ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer),
trace.WithAttributes(
attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"),
attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()),
attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt),
attribute.Int("pipeline.version", envelope.PipelineVersion),
))
w.metrics.ObserveStarted(ctx)
defer func() {
w.metrics.ObserveFinished(ctx, result, w.now().Sub(started))
span.SetAttributes(attribute.String("job.outcome", result))
if resultErr != nil {
span.RecordError(resultErr)
span.SetStatus(codes.Error, result)
if ctx.Err() == nil {
w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr)
}
} else {
w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result)
}
span.End()
}()
lease, err := w.repository.ClaimAttachmentJob(
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
w.config.WorkerID, w.config.LeaseDuration,
)
if err != nil {
result, err = w.resolveClaim(ctx, delivery, envelope, err)
return err
}
if fault == FaultBeforeSideEffect {
result = "injected_failure"
return ErrInjectedFailure
}
processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout)
heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)
heartbeatDone := make(chan error, 1)
go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }()
variants, err := w.process(processingCtx, lease)
cancelProcessing()
cancelHeartbeat()
heartbeatErr := <-heartbeatDone
if heartbeatErr != nil {
return heartbeatErr
}
if err != nil {
result, err = w.resolveProcessingFailure(ctx, delivery, lease, err)
return err
}
if fault == FaultAfterSideEffect {
result = "injected_failure"
return ErrInjectedFailure
}
mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants)
if err != nil {
return fmt.Errorf("complete attachment job: %w", err)
}
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
}
}
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
return fmt.Errorf("unexpected completion result %q", mutation)
}
if fault == FaultAfterCommit {
result = "injected_failure"
return ErrInjectedFailure
}
if err := delivery.Ack(ctx); err != nil {
return err
}
result = "ready"
w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt))
return nil
}
func (w *Worker) maintainLease(
ctx context.Context,
cancelProcessing context.CancelFunc,
lease model.AttachmentJobLease,
) error {
ticker := time.NewTicker(w.config.HeartbeatInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration)
if err != nil {
if ctx.Err() != nil {
return nil
}
cancelProcessing()
return fmt.Errorf("renew attachment job lease: %w", err)
}
if !renewed {
cancelProcessing()
return model.ErrStaleLease
}
}
}
}
func (w *Worker) resolveClaim(
ctx context.Context,
delivery redisstream.Delivery,
envelope redisstream.Envelope,
claimErr error,
) (string, error) {
switch {
case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput):
return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline):
return "duplicate", delivery.Ack(ctx)
case errors.Is(claimErr, model.ErrJobBusy):
return "duplicate", delivery.Ack(ctx)
case errors.Is(claimErr, model.ErrJobNotDue):
return "not_due", nil
case errors.Is(claimErr, model.ErrConflict):
job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String())
if err != nil {
return "transient_error", fmt.Errorf("inspect conflicting job: %w", err)
}
switch job.Status {
case model.AttachmentJobReady:
return "duplicate", delivery.Ack(ctx)
case model.AttachmentJobFailed:
code := "processing_failed"
if job.ErrorCode != nil {
code = *job.ErrorCode
}
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
default:
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
}
default:
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
}
}
func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) {
body, err := w.objects.Get(ctx, lease.Attachment.StagingKey)
if err != nil {
return nil, err
}
defer func() { _ = body.Close() }()
data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > w.config.MaxBytes {
return nil, transform.ErrPayloadTooLarge
}
imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, transform.ErrInvalidImage
}
weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height)
if err != nil {
return nil, err
}
if weight > w.config.MemoryBytes {
return nil, errWorkerCapacity
}
if err := w.memory.Acquire(ctx, weight); err != nil {
return nil, err
}
defer w.memory.Release(weight)
processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType)
if err != nil {
return nil, err
}
variants := []struct {
name string
value transform.Variant
}{
{name: "original", value: processed.Original},
{name: "thumbnail", value: processed.Thumbnail},
{name: "crop", value: processed.Crop},
{name: "optimized", value: processed.Optimized},
}
result := make([]model.Variant, 0, len(variants))
for _, item := range variants {
objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value)
if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil {
return nil, err
}
result = append(result, model.Variant{
Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)),
SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height,
ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion,
})
}
return result, nil
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r contextReader) Read(buffer []byte) (int, error) {
select {
case <-r.ctx.Done():
return 0, r.ctx.Err()
default:
return r.reader.Read(buffer)
}
}
var errWorkerCapacity = errors.New("image exceeds worker memory budget")
func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string {
extension := ".bin"
switch value.ContentType {
case "image/png":
extension = ".png"
case "image/jpeg":
extension = ".jpg"
case "image/webp":
extension = ".webp"
}
return path.Join(
"attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion),
name, value.SHA256+extension,
)
}
func (w *Worker) resolveProcessingFailure(
ctx context.Context,
delivery redisstream.Delivery,
lease model.AttachmentJobLease,
processingErr error,
) (string, error) {
code, permanent := processingError(processingErr)
if permanent || lease.Job.Attempt >= w.config.MaxAttempts {
if !permanent {
code = "retry_exhausted"
}
mutation, err := w.repository.FailAttachmentJob(ctx, lease, code)
if err != nil {
return "transient_error", fmt.Errorf("fail attachment job: %w", err)
}
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
return "duplicate", delivery.Ack(ctx)
}
if mutation != model.JobMutationApplied {
return "transient_error", fmt.Errorf("unexpected failure result %q", mutation)
}
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
}
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
traceContext := telemetry.ContextCarrier(ctx)
mutation, err := w.repository.RetryAttachmentJob(
ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate,
)
if err != nil {
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
}
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
return "duplicate", delivery.Ack(ctx)
}
if mutation != model.JobMutationApplied {
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
}
w.metrics.ObserveRetry(ctx)
return "retry", delivery.Ack(ctx)
}
func processingError(err error) (string, bool) {
switch {
case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia):
return "invalid_image", true
case errors.Is(err, transform.ErrPayloadTooLarge):
return "image_too_large", true
case errors.Is(err, objectstore.ErrNotFound):
return "source_missing", true
case errors.Is(err, errWorkerCapacity):
return "worker_capacity", true
default:
return "processing_failed", false
}
}
func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
delay := w.config.RetryBase
for step := 1; step < attempt && delay < w.config.RetryMax; step++ {
if delay > w.config.RetryMax/2 {
delay = w.config.RetryMax
break
}
delay *= 2
}
if delay > w.config.RetryMax {
delay = w.config.RetryMax
}
var hash uint64
for index := range jobID {
hash = hash*131 + uint64(jobID[index])
}
jitterRange := min(delay/4, w.config.RetryMax-delay)
if jitterRange <= 0 {
return delay
}
jitter := time.Duration(hash % uint64(jitterRange))
if delay > time.Duration(math.MaxInt64)-jitter {
return delay
}
return delay + jitter
}
type nopMetrics struct{}
func (nopMetrics) ObserveStarted(context.Context) {}
func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {}
func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {}
func (nopMetrics) ObserveRetry(context.Context) {}
func (nopMetrics) ObserveReclaimed(context.Context, int) {}