Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
569 lines
14 KiB
Go
569 lines
14 KiB
Go
// Package worker consumes attachment jobs, enforces the lease, produces the
|
|
// contract variants and acknowledges only after the durable result is
|
|
// committed.
|
|
package worker
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
amqp "github.com/rabbitmq/amqp091-go"
|
|
|
|
"mtusi-messenger/internal/broker"
|
|
"mtusi-messenger/internal/domain"
|
|
"mtusi-messenger/internal/failpoint"
|
|
"mtusi-messenger/internal/imagecheck"
|
|
"mtusi-messenger/internal/imageproc"
|
|
"mtusi-messenger/internal/platform/objectstore"
|
|
)
|
|
|
|
type Repository interface {
|
|
ClaimJob(
|
|
ctx context.Context,
|
|
message domain.JobMessage,
|
|
leaseOwner string,
|
|
leaseTTL time.Duration,
|
|
) (domain.ClaimResult, error)
|
|
Heartbeat(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, leaseTTL time.Duration) (bool, error)
|
|
CompleteJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, variants []domain.Variant) (bool, error)
|
|
RetryJob(
|
|
ctx context.Context,
|
|
jobID uuid.UUID,
|
|
attempt int,
|
|
leaseOwner, errorCode string,
|
|
nextAvailable time.Time,
|
|
sameAttempt bool,
|
|
kind string,
|
|
) (failed bool, err error)
|
|
FailJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner, errorCode string) (bool, error)
|
|
MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error
|
|
}
|
|
|
|
type Store interface {
|
|
Open(ctx context.Context, key string) (io.ReadCloser, objectstore.Object, error)
|
|
Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string, metadata map[string]string) error
|
|
Delete(ctx context.Context, key string) error
|
|
}
|
|
|
|
type Consumer interface {
|
|
Consume(ctx context.Context, handler broker.Handler) error
|
|
Connected() bool
|
|
}
|
|
|
|
type Config struct {
|
|
Concurrency int
|
|
LeaseTTL time.Duration
|
|
Heartbeat time.Duration
|
|
JobTimeout time.Duration
|
|
RetryBase time.Duration
|
|
RetryCap time.Duration
|
|
MaxUploadBytes int64
|
|
MaxImagePixels int64
|
|
Owner string
|
|
}
|
|
|
|
type Worker struct {
|
|
repo Repository
|
|
store Store
|
|
config Config
|
|
log *slog.Logger
|
|
consumer Consumer
|
|
|
|
mu sync.RWMutex
|
|
running bool
|
|
}
|
|
|
|
func New(repo Repository, store Store, config Config, log *slog.Logger) *Worker {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
|
|
if config.Owner == "" {
|
|
config.Owner = "worker-" + uuid.NewString()
|
|
}
|
|
|
|
return &Worker{repo: repo, store: store, config: config, log: log}
|
|
}
|
|
|
|
// Run starts Concurrency consumer channels. It returns when the context is
|
|
// cancelled or one consumer fails; the caller recreates the broker connection.
|
|
func (w *Worker) Run(ctx context.Context, consumer Consumer) error {
|
|
w.consumer = consumer
|
|
w.setRunning(true)
|
|
defer w.setRunning(false)
|
|
|
|
runCtx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
var (
|
|
wg sync.WaitGroup
|
|
once sync.Once
|
|
firstErr error
|
|
)
|
|
|
|
for range w.config.Concurrency {
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
|
|
if err := consumer.Consume(runCtx, w.handle); err != nil && runCtx.Err() == nil {
|
|
once.Do(func() {
|
|
firstErr = err
|
|
cancel()
|
|
})
|
|
}
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
return firstErr
|
|
}
|
|
|
|
func (w *Worker) setRunning(value bool) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
|
|
w.running = value
|
|
}
|
|
|
|
func (w *Worker) Ready() (consumerRunning, brokerConnected bool) {
|
|
w.mu.RLock()
|
|
running := w.running
|
|
w.mu.RUnlock()
|
|
|
|
if w.consumer == nil {
|
|
return running, false
|
|
}
|
|
|
|
return running, w.consumer.Connected()
|
|
}
|
|
|
|
func (w *Worker) handle(ctx context.Context, delivery amqp.Delivery) broker.Outcome {
|
|
message, err := domain.ParseJobMessage(delivery.Body)
|
|
if err != nil {
|
|
w.log.WarnContext(ctx, "invalid job payload moved to DLQ", slog.Any("error", err))
|
|
|
|
return broker.Drop
|
|
}
|
|
|
|
result, err := w.repo.ClaimJob(ctx, message, w.config.Owner, w.config.LeaseTTL)
|
|
if err != nil {
|
|
w.log.WarnContext(ctx, "claim job failed", slog.String("job_id", message.JobID.String()), slog.Any("error", err))
|
|
|
|
return broker.Requeue
|
|
}
|
|
|
|
switch result.Outcome {
|
|
case domain.ClaimAcquired:
|
|
w.log.InfoContext(ctx, "job claimed",
|
|
slog.String("job_id", message.JobID.String()),
|
|
slog.String("attachment_id", message.AttachmentID.String()),
|
|
slog.Int("attempt", message.Attempt),
|
|
slog.Int("pipeline_version", message.PipelineVersion),
|
|
)
|
|
case domain.ClaimAlreadyReady, domain.ClaimAlreadyFailed, domain.ClaimSuperseded,
|
|
domain.ClaimStaleAttempt, domain.ClaimFutureAttempt, domain.ClaimBusyLease,
|
|
domain.ClaimNotDue, domain.ClaimVersionStale, domain.ClaimJobNotFound:
|
|
w.log.DebugContext(ctx, "job delivery is a no-op",
|
|
slog.String("job_id", message.JobID.String()),
|
|
slog.String("outcome", string(result.Outcome)),
|
|
)
|
|
|
|
return broker.Ack
|
|
default:
|
|
return broker.Ack
|
|
}
|
|
|
|
return w.process(ctx, result)
|
|
}
|
|
|
|
func (w *Worker) process(
|
|
ctx context.Context,
|
|
result domain.ClaimResult,
|
|
) broker.Outcome {
|
|
job := result.Job
|
|
attachment := result.Attachment
|
|
|
|
failpoint.Check("WORKER_AFTER_CLAIM", job.ID)
|
|
|
|
switch failpoint.Mode("WORKER_FAIL", job.ID) {
|
|
case "permanent":
|
|
return w.fail(ctx, job, permanentError{code: "injected_failure", err: errors.New("injected permanent worker failure")})
|
|
case "transient":
|
|
return w.fail(ctx, job, errors.New("injected transient worker failure"))
|
|
}
|
|
|
|
jobCtx, cancel := context.WithTimeout(ctx, w.config.JobTimeout)
|
|
defer cancel()
|
|
|
|
leaseLost := false
|
|
heartbeatDone := make(chan struct{})
|
|
|
|
go func() {
|
|
defer close(heartbeatDone)
|
|
|
|
ticker := time.NewTicker(w.config.Heartbeat)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-jobCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
ok, err := w.repo.Heartbeat(jobCtx, job.ID, job.Attempt, w.config.Owner, w.config.LeaseTTL)
|
|
if err != nil {
|
|
// A DB blip is not proof that the lease is gone. Keep trying;
|
|
// the fencing CAS still prevents a stale commit.
|
|
w.log.WarnContext(jobCtx, "heartbeat failed", slog.Any("error", err))
|
|
|
|
continue
|
|
}
|
|
if !ok {
|
|
leaseLost = true
|
|
cancel()
|
|
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
// The hook runs after the heartbeat starts: it can be paused to test a
|
|
// genuinely stuck (not merely slow) worker without consuming every attempt.
|
|
failpoint.Sleep("WORKER_SLEEP_AFTER_CLAIM", job.ID)
|
|
|
|
watchdog := time.AfterFunc(w.config.JobTimeout+30*time.Second, func() {
|
|
w.log.ErrorContext(ctx, "worker job watchdog fired",
|
|
slog.String("job_id", job.ID.String()),
|
|
slog.String("attachment_id", job.AttachmentID.String()),
|
|
)
|
|
os.Exit(70)
|
|
})
|
|
defer watchdog.Stop()
|
|
|
|
err := w.runJob(jobCtx, job, attachment)
|
|
cancel()
|
|
<-heartbeatDone
|
|
|
|
if leaseLost {
|
|
w.log.WarnContext(ctx, "lease lost; reaper will requeue",
|
|
slog.String("job_id", job.ID.String()),
|
|
)
|
|
|
|
return broker.Ack
|
|
}
|
|
|
|
if err == nil {
|
|
failpoint.Check("WORKER_AFTER_COMMIT", job.ID)
|
|
|
|
return broker.Ack
|
|
}
|
|
|
|
return w.fail(ctx, job, err)
|
|
}
|
|
|
|
func (w *Worker) runJob(
|
|
ctx context.Context,
|
|
job domain.Job,
|
|
attachment domain.Attachment,
|
|
) error {
|
|
params, ok := imageproc.Pipeline(job.PipelineVersion)
|
|
if !ok {
|
|
return permanentError{code: "unsupported_pipeline", err: fmt.Errorf("pipeline %d", job.PipelineVersion)}
|
|
}
|
|
|
|
sourceKey := job.SourceKey
|
|
if sourceKey == "" {
|
|
sourceKey = attachment.StagingKey
|
|
}
|
|
if sourceKey == "" {
|
|
sourceKey = attachment.ObjectKey
|
|
}
|
|
|
|
file, size, sum, err := w.spoolSource(ctx, sourceKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer func() { _ = file.Close() }()
|
|
defer func() { _ = os.Remove(file.Name()) }()
|
|
|
|
if size != attachment.SizeBytes || sum != attachment.SHA256 {
|
|
return permanentError{code: "source_corrupt", err: errors.New("source bytes do not match metadata")}
|
|
}
|
|
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
return fmt.Errorf("rewind source: %w", err)
|
|
}
|
|
|
|
if _, err := imagecheck.Validate(file, w.config.MaxImagePixels); err != nil {
|
|
return permanentError{code: "invalid_image", err: err}
|
|
}
|
|
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
return fmt.Errorf("rewind source: %w", err)
|
|
}
|
|
|
|
img, _, err := imageproc.Load(file.Name(), w.config.MaxImagePixels)
|
|
if err != nil {
|
|
return permanentError{code: "invalid_image", err: err}
|
|
}
|
|
|
|
if sourceKey != attachment.ObjectKey {
|
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
return fmt.Errorf("rewind original: %w", err)
|
|
}
|
|
|
|
if err := w.store.Put(
|
|
ctx, attachment.ObjectKey, file, size, attachment.ContentType,
|
|
map[string]string{"sha256": sum},
|
|
); err != nil {
|
|
return fmt.Errorf("publish original: %w", err)
|
|
}
|
|
}
|
|
|
|
variants := make([]domain.Variant, 0, 3)
|
|
|
|
generated := []func() (imageproc.Image, error){
|
|
func() (imageproc.Image, error) { return imageproc.Thumbnail(img, params) },
|
|
func() (imageproc.Image, error) { return imageproc.Crop(img, params) },
|
|
func() (imageproc.Image, error) { return imageproc.Optimized(img, params) },
|
|
}
|
|
|
|
for idx, generate := range generated {
|
|
variantImage, err := generate()
|
|
if err != nil {
|
|
return permanentError{code: "processing_failed", err: err}
|
|
}
|
|
|
|
key := domain.VariantObjectKey(attachment.ChatID, attachment.ID, job.PipelineVersion, variantImage.Name)
|
|
|
|
if err := w.store.Put(
|
|
ctx, key, bytes.NewReader(variantImage.Data), variantImage.SizeBytes,
|
|
variantImage.ContentType, map[string]string{"sha256": variantImage.SHA256},
|
|
); err != nil {
|
|
return fmt.Errorf("publish variant %s: %w", variantImage.Name, err)
|
|
}
|
|
|
|
if idx == 0 {
|
|
failpoint.Check("WORKER_AFTER_FIRST_VARIANT", job.ID)
|
|
}
|
|
|
|
variants = append(variants, domain.Variant{
|
|
Name: variantImage.Name,
|
|
ContentType: variantImage.ContentType,
|
|
SizeBytes: variantImage.SizeBytes,
|
|
Width: variantImage.Width,
|
|
Height: variantImage.Height,
|
|
SHA256: variantImage.SHA256,
|
|
ObjectKey: key,
|
|
})
|
|
}
|
|
|
|
failpoint.Check("WORKER_AFTER_PUT", job.ID)
|
|
|
|
completed, err := w.repo.CompleteJob(ctx, job.ID, job.Attempt, w.config.Owner, variants)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !completed {
|
|
w.log.WarnContext(ctx, "completion CAS lost", slog.String("job_id", job.ID.String()))
|
|
|
|
return nil
|
|
}
|
|
|
|
w.log.InfoContext(ctx, "attachment ready",
|
|
slog.String("job_id", job.ID.String()),
|
|
slog.String("attachment_id", attachment.ID.String()),
|
|
slog.Int("pipeline_version", job.PipelineVersion),
|
|
)
|
|
|
|
if strings.HasPrefix(sourceKey, "staging/") && sourceKey != attachment.ObjectKey {
|
|
if err := w.store.Delete(ctx, sourceKey); err != nil && !errors.Is(err, domain.ErrNotFound) {
|
|
w.log.WarnContext(ctx, "staging delete failed", slog.String("object_key", sourceKey), slog.Any("error", err))
|
|
|
|
return nil
|
|
}
|
|
|
|
if sourceKey == attachment.StagingKey {
|
|
if err := w.repo.MarkStagingDeleted(ctx, attachment.ID, sourceKey); err != nil {
|
|
w.log.WarnContext(ctx, "clear staging projection failed", slog.Any("error", err))
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) spoolSource(ctx context.Context, key string) (*os.File, int64, string, error) {
|
|
body, _, err := w.store.Open(ctx, key)
|
|
if err != nil {
|
|
if errors.Is(err, domain.ErrNotFound) {
|
|
return nil, 0, "", permanentError{code: "source_missing", err: err}
|
|
}
|
|
|
|
return nil, 0, "", fmt.Errorf("open source: %w", err)
|
|
}
|
|
defer body.Close()
|
|
|
|
file, err := os.CreateTemp("", "lab5-worker-*")
|
|
if err != nil {
|
|
return nil, 0, "", fmt.Errorf("create worker spool: %w", err)
|
|
}
|
|
|
|
cleanup := func() {
|
|
_ = file.Close()
|
|
_ = os.Remove(file.Name())
|
|
}
|
|
|
|
hasher := sha256.New()
|
|
limited := io.LimitReader(body, w.config.MaxUploadBytes+1)
|
|
buffer := make([]byte, 64<<10)
|
|
|
|
size, err := io.CopyBuffer(io.MultiWriter(file, hasher), limited, buffer)
|
|
if err != nil {
|
|
cleanup()
|
|
|
|
return nil, 0, "", fmt.Errorf("copy source: %w", err)
|
|
}
|
|
|
|
if size > w.config.MaxUploadBytes {
|
|
cleanup()
|
|
|
|
return nil, 0, "", permanentError{code: "source_too_large", err: fmt.Errorf("source exceeds %d bytes", w.config.MaxUploadBytes)}
|
|
}
|
|
|
|
return file, size, hex.EncodeToString(hasher.Sum(nil)), nil
|
|
}
|
|
|
|
func (w *Worker) fail(ctx context.Context, job domain.Job, err error) broker.Outcome {
|
|
if ctx.Err() != nil {
|
|
// Parent cancellation means graceful shutdown. Keep the attempt number:
|
|
// an operator restart must not consume the processing budget. The job
|
|
// context is already cancelled, so use a short detached context for the
|
|
// durable requeue.
|
|
retryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
defer cancel()
|
|
|
|
failed, retryErr := w.repo.RetryJob(
|
|
retryCtx, job.ID, job.Attempt, w.config.Owner, "shutdown",
|
|
time.Now().Add(time.Second), true, "shutdown",
|
|
)
|
|
if retryErr != nil {
|
|
return broker.Requeue
|
|
}
|
|
|
|
_ = failed
|
|
|
|
return broker.Ack
|
|
}
|
|
|
|
code := "processing_failed"
|
|
permanent := false
|
|
|
|
var typed permanentError
|
|
if errors.As(err, &typed) {
|
|
code = typed.code
|
|
permanent = true
|
|
}
|
|
|
|
if errors.Is(err, domain.ErrValidation) || errors.Is(err, domain.ErrUnsupportedMedia) ||
|
|
errors.Is(err, domain.ErrTooLarge) || errors.Is(err, domain.ErrConflict) {
|
|
permanent = true
|
|
if code == "processing_failed" {
|
|
code = "invalid_image"
|
|
}
|
|
}
|
|
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
code = "processing_timeout"
|
|
permanent = false
|
|
}
|
|
|
|
if permanent {
|
|
w.log.WarnContext(ctx, "job permanently failed",
|
|
slog.String("job_id", job.ID.String()),
|
|
slog.String("error_code", code),
|
|
slog.Any("error", err),
|
|
)
|
|
|
|
ok, failErr := w.repo.FailJob(ctx, job.ID, job.Attempt, w.config.Owner, code)
|
|
if failErr != nil {
|
|
return broker.Requeue
|
|
}
|
|
if !ok {
|
|
return broker.Ack
|
|
}
|
|
|
|
return broker.Drop
|
|
}
|
|
|
|
next := time.Now().Add(w.backoff(job.Attempt))
|
|
w.log.WarnContext(ctx, "job retry scheduled",
|
|
slog.String("job_id", job.ID.String()),
|
|
slog.String("error_code", code),
|
|
slog.Time("available_at", next),
|
|
slog.Any("error", err),
|
|
)
|
|
|
|
failed, retryErr := w.repo.RetryJob(
|
|
ctx, job.ID, job.Attempt, w.config.Owner, code, next, false, "retry",
|
|
)
|
|
if retryErr != nil {
|
|
return broker.Requeue
|
|
}
|
|
|
|
if failed {
|
|
return broker.Drop
|
|
}
|
|
|
|
return broker.Ack
|
|
}
|
|
|
|
func (w *Worker) backoff(attempt int) time.Duration {
|
|
if attempt < 1 {
|
|
attempt = 1
|
|
}
|
|
|
|
capDelay := w.config.RetryCap
|
|
base := w.config.RetryBase
|
|
|
|
for range attempt - 1 {
|
|
base *= 2
|
|
if base >= capDelay {
|
|
base = capDelay
|
|
|
|
break
|
|
}
|
|
}
|
|
|
|
if base <= 0 {
|
|
return time.Second
|
|
}
|
|
|
|
return time.Duration(rand.Int64N(int64(base))) + time.Millisecond
|
|
}
|
|
|
|
type permanentError struct {
|
|
code string
|
|
err error
|
|
}
|
|
|
|
func (e permanentError) Error() string { return e.code + ": " + e.err.Error() }
|
|
func (e permanentError) Unwrap() error { return e.err }
|