forked from mtusi2026/mtusi2026lab6
544 lines
18 KiB
Go
544 lines
18 KiB
Go
package worker
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"log/slog"
|
|
"math"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/sync/semaphore"
|
|
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry"
|
|
)
|
|
|
|
type Repository interface {
|
|
AttachmentJob(context.Context, string) (model.AttachmentJob, error)
|
|
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
|
|
RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error)
|
|
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
|
|
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error)
|
|
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
|
|
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
|
|
}
|
|
|
|
type Broker interface {
|
|
Read(context.Context) ([]redisstream.Delivery, error)
|
|
Reclaim(context.Context, string) ([]redisstream.Delivery, string, error)
|
|
}
|
|
|
|
type ObjectStore interface {
|
|
Get(context.Context, string) (io.ReadCloser, error)
|
|
Put(context.Context, string, string, []byte) error
|
|
Delete(context.Context, string) error
|
|
}
|
|
|
|
type ImageProcessor interface {
|
|
Process(context.Context, []byte, string) (transform.Result, error)
|
|
}
|
|
|
|
type Metrics interface {
|
|
ObserveStarted(context.Context)
|
|
ObserveFinished(context.Context, string, time.Duration)
|
|
ObserveTimeToReady(context.Context, time.Duration)
|
|
ObserveRetry(context.Context)
|
|
ObserveReclaimed(context.Context, int)
|
|
}
|
|
|
|
type Config struct {
|
|
WorkerID string
|
|
Concurrency int
|
|
LeaseDuration time.Duration
|
|
HeartbeatInterval time.Duration
|
|
ProcessingTimeout time.Duration
|
|
ReclaimInterval time.Duration
|
|
MaxAttempts int
|
|
RetryBase time.Duration
|
|
RetryMax time.Duration
|
|
MaxBytes int64
|
|
MaxPixels int64
|
|
MemoryBytes int64
|
|
}
|
|
|
|
type FaultPoint string
|
|
|
|
const (
|
|
FaultNone FaultPoint = ""
|
|
FaultBeforeSideEffect FaultPoint = "before_side_effect"
|
|
FaultAfterSideEffect FaultPoint = "after_side_effect"
|
|
FaultAfterCommit FaultPoint = "after_commit"
|
|
)
|
|
|
|
var ErrInjectedFailure = errors.New("injected worker failure")
|
|
|
|
type Worker struct {
|
|
logger *slog.Logger
|
|
repository Repository
|
|
broker Broker
|
|
objects ObjectStore
|
|
processor ImageProcessor
|
|
metrics Metrics
|
|
config Config
|
|
memory *semaphore.Weighted
|
|
now func() time.Time
|
|
}
|
|
|
|
func New(
|
|
logger *slog.Logger,
|
|
repository Repository,
|
|
broker Broker,
|
|
objects ObjectStore,
|
|
processor ImageProcessor,
|
|
metrics Metrics,
|
|
config Config,
|
|
) (*Worker, error) {
|
|
if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil {
|
|
return nil, errors.New("worker dependencies are required")
|
|
}
|
|
if err := validateConfig(config); err != nil {
|
|
return nil, err
|
|
}
|
|
if metrics == nil {
|
|
metrics = nopMetrics{}
|
|
}
|
|
return &Worker{
|
|
logger: logger, repository: repository, broker: broker, objects: objects,
|
|
processor: processor, metrics: metrics, config: config,
|
|
memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now,
|
|
}, nil
|
|
}
|
|
|
|
func validateConfig(config Config) error {
|
|
if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 ||
|
|
config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration ||
|
|
config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration ||
|
|
config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 ||
|
|
config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 {
|
|
return errors.New("invalid worker configuration")
|
|
}
|
|
const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128
|
|
if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 ||
|
|
config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes {
|
|
return errors.New("worker memory must fit one maximum image")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) Run(ctx context.Context) error {
|
|
lastReclaim := time.Time{}
|
|
reclaimCursor := "0-0"
|
|
for ctx.Err() == nil {
|
|
deliveries, err := w.broker.Read(ctx)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
w.processBatch(ctx, deliveries)
|
|
if w.now().Sub(lastReclaim) < w.config.ReclaimInterval {
|
|
continue
|
|
}
|
|
recovery, err := w.repository.RecoverExpiredAttachmentJobs(
|
|
ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase),
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("recover expired attachment jobs: %w", err)
|
|
}
|
|
if recovery.Requeued > 0 || recovery.Failed > 0 {
|
|
w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed)
|
|
}
|
|
deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
w.metrics.ObserveReclaimed(ctx, len(deliveries))
|
|
w.processBatch(ctx, deliveries)
|
|
reclaimCursor = next
|
|
if reclaimCursor == "" {
|
|
reclaimCursor = "0-0"
|
|
}
|
|
lastReclaim = w.now()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) {
|
|
limit := make(chan struct{}, w.config.Concurrency)
|
|
var group sync.WaitGroup
|
|
for _, delivery := range deliveries {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
limit <- struct{}{}
|
|
group.Add(1)
|
|
go func(delivery redisstream.Delivery) {
|
|
defer group.Done()
|
|
defer func() { <-limit }()
|
|
_ = w.ProcessDelivery(ctx, delivery, FaultNone)
|
|
}(delivery)
|
|
}
|
|
group.Wait()
|
|
}
|
|
|
|
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) {
|
|
started := w.now()
|
|
result := "transient_error"
|
|
if delivery.DecodeError != nil {
|
|
w.metrics.ObserveStarted(ctx)
|
|
defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }()
|
|
result = "invalid_job"
|
|
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
|
}
|
|
envelope := delivery.Envelope
|
|
ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{
|
|
Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate,
|
|
})
|
|
ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer),
|
|
trace.WithAttributes(
|
|
attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"),
|
|
attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()),
|
|
attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt),
|
|
attribute.Int("pipeline.version", envelope.PipelineVersion),
|
|
))
|
|
w.metrics.ObserveStarted(ctx)
|
|
defer func() {
|
|
w.metrics.ObserveFinished(ctx, result, w.now().Sub(started))
|
|
span.SetAttributes(attribute.String("job.outcome", result))
|
|
if resultErr != nil {
|
|
span.RecordError(resultErr)
|
|
span.SetStatus(codes.Error, result)
|
|
if ctx.Err() == nil {
|
|
w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr)
|
|
}
|
|
} else {
|
|
w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result)
|
|
}
|
|
span.End()
|
|
}()
|
|
|
|
lease, err := w.repository.ClaimAttachmentJob(
|
|
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
|
|
w.config.WorkerID, w.config.LeaseDuration,
|
|
)
|
|
if err != nil {
|
|
result, err = w.resolveClaim(ctx, delivery, envelope, err)
|
|
return err
|
|
}
|
|
if fault == FaultBeforeSideEffect {
|
|
result = "injected_failure"
|
|
return ErrInjectedFailure
|
|
}
|
|
|
|
processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout)
|
|
heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)
|
|
heartbeatDone := make(chan error, 1)
|
|
go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }()
|
|
variants, err := w.process(processingCtx, lease)
|
|
cancelProcessing()
|
|
cancelHeartbeat()
|
|
heartbeatErr := <-heartbeatDone
|
|
if heartbeatErr != nil {
|
|
return heartbeatErr
|
|
}
|
|
if err != nil {
|
|
result, err = w.resolveProcessingFailure(ctx, delivery, lease, err)
|
|
return err
|
|
}
|
|
if fault == FaultAfterSideEffect {
|
|
result = "injected_failure"
|
|
return ErrInjectedFailure
|
|
}
|
|
mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants)
|
|
if err != nil {
|
|
return fmt.Errorf("complete attachment job: %w", err)
|
|
}
|
|
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
|
|
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
|
|
w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
|
|
}
|
|
}
|
|
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
|
|
return fmt.Errorf("unexpected completion result %q", mutation)
|
|
}
|
|
if fault == FaultAfterCommit {
|
|
result = "injected_failure"
|
|
return ErrInjectedFailure
|
|
}
|
|
if err := delivery.Ack(ctx); err != nil {
|
|
return err
|
|
}
|
|
result = "ready"
|
|
w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt))
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) maintainLease(
|
|
ctx context.Context,
|
|
cancelProcessing context.CancelFunc,
|
|
lease model.AttachmentJobLease,
|
|
) error {
|
|
ticker := time.NewTicker(w.config.HeartbeatInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-ticker.C:
|
|
renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
cancelProcessing()
|
|
return fmt.Errorf("renew attachment job lease: %w", err)
|
|
}
|
|
if !renewed {
|
|
cancelProcessing()
|
|
return model.ErrStaleLease
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *Worker) resolveClaim(
|
|
ctx context.Context,
|
|
delivery redisstream.Delivery,
|
|
envelope redisstream.Envelope,
|
|
claimErr error,
|
|
) (string, error) {
|
|
switch {
|
|
case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput):
|
|
return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
|
case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline):
|
|
return "duplicate", delivery.Ack(ctx)
|
|
case errors.Is(claimErr, model.ErrJobBusy):
|
|
return "duplicate", delivery.Ack(ctx)
|
|
case errors.Is(claimErr, model.ErrJobNotDue):
|
|
return "not_due", nil
|
|
case errors.Is(claimErr, model.ErrConflict):
|
|
job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String())
|
|
if err != nil {
|
|
return "transient_error", fmt.Errorf("inspect conflicting job: %w", err)
|
|
}
|
|
switch job.Status {
|
|
case model.AttachmentJobReady:
|
|
return "duplicate", delivery.Ack(ctx)
|
|
case model.AttachmentJobFailed:
|
|
code := "processing_failed"
|
|
if job.ErrorCode != nil {
|
|
code = *job.ErrorCode
|
|
}
|
|
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
|
default:
|
|
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
|
}
|
|
default:
|
|
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
|
}
|
|
}
|
|
|
|
func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) {
|
|
body, err := w.objects.Get(ctx, lease.Attachment.StagingKey)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = body.Close() }()
|
|
data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if int64(len(data)) > w.config.MaxBytes {
|
|
return nil, transform.ErrPayloadTooLarge
|
|
}
|
|
imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)})
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
return nil, transform.ErrInvalidImage
|
|
}
|
|
weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if weight > w.config.MemoryBytes {
|
|
return nil, errWorkerCapacity
|
|
}
|
|
if err := w.memory.Acquire(ctx, weight); err != nil {
|
|
return nil, err
|
|
}
|
|
defer w.memory.Release(weight)
|
|
|
|
processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
variants := []struct {
|
|
name string
|
|
value transform.Variant
|
|
}{
|
|
{name: "original", value: processed.Original},
|
|
{name: "thumbnail", value: processed.Thumbnail},
|
|
{name: "crop", value: processed.Crop},
|
|
{name: "optimized", value: processed.Optimized},
|
|
}
|
|
result := make([]model.Variant, 0, len(variants))
|
|
for _, item := range variants {
|
|
objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value)
|
|
if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil {
|
|
return nil, err
|
|
}
|
|
result = append(result, model.Variant{
|
|
Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)),
|
|
SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height,
|
|
ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion,
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
type contextReader struct {
|
|
ctx context.Context
|
|
reader io.Reader
|
|
}
|
|
|
|
func (r contextReader) Read(buffer []byte) (int, error) {
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return 0, r.ctx.Err()
|
|
default:
|
|
return r.reader.Read(buffer)
|
|
}
|
|
}
|
|
|
|
var errWorkerCapacity = errors.New("image exceeds worker memory budget")
|
|
|
|
func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string {
|
|
extension := ".bin"
|
|
switch value.ContentType {
|
|
case "image/png":
|
|
extension = ".png"
|
|
case "image/jpeg":
|
|
extension = ".jpg"
|
|
case "image/webp":
|
|
extension = ".webp"
|
|
}
|
|
return path.Join(
|
|
"attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion),
|
|
name, value.SHA256+extension,
|
|
)
|
|
}
|
|
|
|
func (w *Worker) resolveProcessingFailure(
|
|
ctx context.Context,
|
|
delivery redisstream.Delivery,
|
|
lease model.AttachmentJobLease,
|
|
processingErr error,
|
|
) (string, error) {
|
|
code, permanent := processingError(processingErr)
|
|
if permanent || lease.Job.Attempt >= w.config.MaxAttempts {
|
|
if !permanent {
|
|
code = "retry_exhausted"
|
|
}
|
|
mutation, err := w.repository.FailAttachmentJob(ctx, lease, code)
|
|
if err != nil {
|
|
return "transient_error", fmt.Errorf("fail attachment job: %w", err)
|
|
}
|
|
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
|
|
return "duplicate", delivery.Ack(ctx)
|
|
}
|
|
if mutation != model.JobMutationApplied {
|
|
return "transient_error", fmt.Errorf("unexpected failure result %q", mutation)
|
|
}
|
|
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
|
}
|
|
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
|
|
traceContext := telemetry.ContextCarrier(ctx)
|
|
mutation, err := w.repository.RetryAttachmentJob(
|
|
ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate,
|
|
)
|
|
if err != nil {
|
|
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
|
|
}
|
|
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
|
|
return "duplicate", delivery.Ack(ctx)
|
|
}
|
|
if mutation != model.JobMutationApplied {
|
|
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
|
|
}
|
|
w.metrics.ObserveRetry(ctx)
|
|
return "retry", delivery.Ack(ctx)
|
|
}
|
|
|
|
func processingError(err error) (string, bool) {
|
|
switch {
|
|
case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia):
|
|
return "invalid_image", true
|
|
case errors.Is(err, transform.ErrPayloadTooLarge):
|
|
return "image_too_large", true
|
|
case errors.Is(err, objectstore.ErrNotFound):
|
|
return "source_missing", true
|
|
case errors.Is(err, errWorkerCapacity):
|
|
return "worker_capacity", true
|
|
default:
|
|
return "processing_failed", false
|
|
}
|
|
}
|
|
|
|
func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
|
|
delay := w.config.RetryBase
|
|
for step := 1; step < attempt && delay < w.config.RetryMax; step++ {
|
|
if delay > w.config.RetryMax/2 {
|
|
delay = w.config.RetryMax
|
|
break
|
|
}
|
|
delay *= 2
|
|
}
|
|
if delay > w.config.RetryMax {
|
|
delay = w.config.RetryMax
|
|
}
|
|
var hash uint64
|
|
for index := range jobID {
|
|
hash = hash*131 + uint64(jobID[index])
|
|
}
|
|
jitterRange := min(delay/4, w.config.RetryMax-delay)
|
|
if jitterRange <= 0 {
|
|
return delay
|
|
}
|
|
jitter := time.Duration(hash % uint64(jitterRange))
|
|
if delay > time.Duration(math.MaxInt64)-jitter {
|
|
return delay
|
|
}
|
|
return delay + jitter
|
|
}
|
|
|
|
type nopMetrics struct{}
|
|
|
|
func (nopMetrics) ObserveStarted(context.Context) {}
|
|
func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {}
|
|
func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {}
|
|
func (nopMetrics) ObserveRetry(context.Context) {}
|
|
func (nopMetrics) ObserveReclaimed(context.Context, int) {}
|