Files
mtusi2026lab5/internal/worker/worker.go
T
Vladislav Kunetskiy 01b885e846 lab5: очереди и асинхронная обработка изображений
Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
2026-10-31 14:34:24 +03:00

569 lines
14 KiB
Go

// Package worker consumes attachment jobs, enforces the lease, produces the
// contract variants and acknowledges only after the durable result is
// committed.
package worker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"os"
"strings"
"sync"
"time"
"github.com/google/uuid"
amqp "github.com/rabbitmq/amqp091-go"
"mtusi-messenger/internal/broker"
"mtusi-messenger/internal/domain"
"mtusi-messenger/internal/failpoint"
"mtusi-messenger/internal/imagecheck"
"mtusi-messenger/internal/imageproc"
"mtusi-messenger/internal/platform/objectstore"
)
type Repository interface {
ClaimJob(
ctx context.Context,
message domain.JobMessage,
leaseOwner string,
leaseTTL time.Duration,
) (domain.ClaimResult, error)
Heartbeat(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, leaseTTL time.Duration) (bool, error)
CompleteJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner string, variants []domain.Variant) (bool, error)
RetryJob(
ctx context.Context,
jobID uuid.UUID,
attempt int,
leaseOwner, errorCode string,
nextAvailable time.Time,
sameAttempt bool,
kind string,
) (failed bool, err error)
FailJob(ctx context.Context, jobID uuid.UUID, attempt int, leaseOwner, errorCode string) (bool, error)
MarkStagingDeleted(ctx context.Context, attachmentID uuid.UUID, stagingKey string) error
}
type Store interface {
Open(ctx context.Context, key string) (io.ReadCloser, objectstore.Object, error)
Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string, metadata map[string]string) error
Delete(ctx context.Context, key string) error
}
type Consumer interface {
Consume(ctx context.Context, handler broker.Handler) error
Connected() bool
}
type Config struct {
Concurrency int
LeaseTTL time.Duration
Heartbeat time.Duration
JobTimeout time.Duration
RetryBase time.Duration
RetryCap time.Duration
MaxUploadBytes int64
MaxImagePixels int64
Owner string
}
type Worker struct {
repo Repository
store Store
config Config
log *slog.Logger
consumer Consumer
mu sync.RWMutex
running bool
}
func New(repo Repository, store Store, config Config, log *slog.Logger) *Worker {
if log == nil {
log = slog.Default()
}
if config.Owner == "" {
config.Owner = "worker-" + uuid.NewString()
}
return &Worker{repo: repo, store: store, config: config, log: log}
}
// Run starts Concurrency consumer channels. It returns when the context is
// cancelled or one consumer fails; the caller recreates the broker connection.
func (w *Worker) Run(ctx context.Context, consumer Consumer) error {
w.consumer = consumer
w.setRunning(true)
defer w.setRunning(false)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
var (
wg sync.WaitGroup
once sync.Once
firstErr error
)
for range w.config.Concurrency {
wg.Add(1)
go func() {
defer wg.Done()
if err := consumer.Consume(runCtx, w.handle); err != nil && runCtx.Err() == nil {
once.Do(func() {
firstErr = err
cancel()
})
}
}()
}
wg.Wait()
return firstErr
}
func (w *Worker) setRunning(value bool) {
w.mu.Lock()
defer w.mu.Unlock()
w.running = value
}
func (w *Worker) Ready() (consumerRunning, brokerConnected bool) {
w.mu.RLock()
running := w.running
w.mu.RUnlock()
if w.consumer == nil {
return running, false
}
return running, w.consumer.Connected()
}
func (w *Worker) handle(ctx context.Context, delivery amqp.Delivery) broker.Outcome {
message, err := domain.ParseJobMessage(delivery.Body)
if err != nil {
w.log.WarnContext(ctx, "invalid job payload moved to DLQ", slog.Any("error", err))
return broker.Drop
}
result, err := w.repo.ClaimJob(ctx, message, w.config.Owner, w.config.LeaseTTL)
if err != nil {
w.log.WarnContext(ctx, "claim job failed", slog.String("job_id", message.JobID.String()), slog.Any("error", err))
return broker.Requeue
}
switch result.Outcome {
case domain.ClaimAcquired:
w.log.InfoContext(ctx, "job claimed",
slog.String("job_id", message.JobID.String()),
slog.String("attachment_id", message.AttachmentID.String()),
slog.Int("attempt", message.Attempt),
slog.Int("pipeline_version", message.PipelineVersion),
)
case domain.ClaimAlreadyReady, domain.ClaimAlreadyFailed, domain.ClaimSuperseded,
domain.ClaimStaleAttempt, domain.ClaimFutureAttempt, domain.ClaimBusyLease,
domain.ClaimNotDue, domain.ClaimVersionStale, domain.ClaimJobNotFound:
w.log.DebugContext(ctx, "job delivery is a no-op",
slog.String("job_id", message.JobID.String()),
slog.String("outcome", string(result.Outcome)),
)
return broker.Ack
default:
return broker.Ack
}
return w.process(ctx, result)
}
func (w *Worker) process(
ctx context.Context,
result domain.ClaimResult,
) broker.Outcome {
job := result.Job
attachment := result.Attachment
failpoint.Check("WORKER_AFTER_CLAIM", job.ID)
switch failpoint.Mode("WORKER_FAIL", job.ID) {
case "permanent":
return w.fail(ctx, job, permanentError{code: "injected_failure", err: errors.New("injected permanent worker failure")})
case "transient":
return w.fail(ctx, job, errors.New("injected transient worker failure"))
}
jobCtx, cancel := context.WithTimeout(ctx, w.config.JobTimeout)
defer cancel()
leaseLost := false
heartbeatDone := make(chan struct{})
go func() {
defer close(heartbeatDone)
ticker := time.NewTicker(w.config.Heartbeat)
defer ticker.Stop()
for {
select {
case <-jobCtx.Done():
return
case <-ticker.C:
ok, err := w.repo.Heartbeat(jobCtx, job.ID, job.Attempt, w.config.Owner, w.config.LeaseTTL)
if err != nil {
// A DB blip is not proof that the lease is gone. Keep trying;
// the fencing CAS still prevents a stale commit.
w.log.WarnContext(jobCtx, "heartbeat failed", slog.Any("error", err))
continue
}
if !ok {
leaseLost = true
cancel()
return
}
}
}
}()
// The hook runs after the heartbeat starts: it can be paused to test a
// genuinely stuck (not merely slow) worker without consuming every attempt.
failpoint.Sleep("WORKER_SLEEP_AFTER_CLAIM", job.ID)
watchdog := time.AfterFunc(w.config.JobTimeout+30*time.Second, func() {
w.log.ErrorContext(ctx, "worker job watchdog fired",
slog.String("job_id", job.ID.String()),
slog.String("attachment_id", job.AttachmentID.String()),
)
os.Exit(70)
})
defer watchdog.Stop()
err := w.runJob(jobCtx, job, attachment)
cancel()
<-heartbeatDone
if leaseLost {
w.log.WarnContext(ctx, "lease lost; reaper will requeue",
slog.String("job_id", job.ID.String()),
)
return broker.Ack
}
if err == nil {
failpoint.Check("WORKER_AFTER_COMMIT", job.ID)
return broker.Ack
}
return w.fail(ctx, job, err)
}
func (w *Worker) runJob(
ctx context.Context,
job domain.Job,
attachment domain.Attachment,
) error {
params, ok := imageproc.Pipeline(job.PipelineVersion)
if !ok {
return permanentError{code: "unsupported_pipeline", err: fmt.Errorf("pipeline %d", job.PipelineVersion)}
}
sourceKey := job.SourceKey
if sourceKey == "" {
sourceKey = attachment.StagingKey
}
if sourceKey == "" {
sourceKey = attachment.ObjectKey
}
file, size, sum, err := w.spoolSource(ctx, sourceKey)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
defer func() { _ = os.Remove(file.Name()) }()
if size != attachment.SizeBytes || sum != attachment.SHA256 {
return permanentError{code: "source_corrupt", err: errors.New("source bytes do not match metadata")}
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("rewind source: %w", err)
}
if _, err := imagecheck.Validate(file, w.config.MaxImagePixels); err != nil {
return permanentError{code: "invalid_image", err: err}
}
if _, err := file.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("rewind source: %w", err)
}
img, _, err := imageproc.Load(file.Name(), w.config.MaxImagePixels)
if err != nil {
return permanentError{code: "invalid_image", err: err}
}
if sourceKey != attachment.ObjectKey {
if _, err := file.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("rewind original: %w", err)
}
if err := w.store.Put(
ctx, attachment.ObjectKey, file, size, attachment.ContentType,
map[string]string{"sha256": sum},
); err != nil {
return fmt.Errorf("publish original: %w", err)
}
}
variants := make([]domain.Variant, 0, 3)
generated := []func() (imageproc.Image, error){
func() (imageproc.Image, error) { return imageproc.Thumbnail(img, params) },
func() (imageproc.Image, error) { return imageproc.Crop(img, params) },
func() (imageproc.Image, error) { return imageproc.Optimized(img, params) },
}
for idx, generate := range generated {
variantImage, err := generate()
if err != nil {
return permanentError{code: "processing_failed", err: err}
}
key := domain.VariantObjectKey(attachment.ChatID, attachment.ID, job.PipelineVersion, variantImage.Name)
if err := w.store.Put(
ctx, key, bytes.NewReader(variantImage.Data), variantImage.SizeBytes,
variantImage.ContentType, map[string]string{"sha256": variantImage.SHA256},
); err != nil {
return fmt.Errorf("publish variant %s: %w", variantImage.Name, err)
}
if idx == 0 {
failpoint.Check("WORKER_AFTER_FIRST_VARIANT", job.ID)
}
variants = append(variants, domain.Variant{
Name: variantImage.Name,
ContentType: variantImage.ContentType,
SizeBytes: variantImage.SizeBytes,
Width: variantImage.Width,
Height: variantImage.Height,
SHA256: variantImage.SHA256,
ObjectKey: key,
})
}
failpoint.Check("WORKER_AFTER_PUT", job.ID)
completed, err := w.repo.CompleteJob(ctx, job.ID, job.Attempt, w.config.Owner, variants)
if err != nil {
return err
}
if !completed {
w.log.WarnContext(ctx, "completion CAS lost", slog.String("job_id", job.ID.String()))
return nil
}
w.log.InfoContext(ctx, "attachment ready",
slog.String("job_id", job.ID.String()),
slog.String("attachment_id", attachment.ID.String()),
slog.Int("pipeline_version", job.PipelineVersion),
)
if strings.HasPrefix(sourceKey, "staging/") && sourceKey != attachment.ObjectKey {
if err := w.store.Delete(ctx, sourceKey); err != nil && !errors.Is(err, domain.ErrNotFound) {
w.log.WarnContext(ctx, "staging delete failed", slog.String("object_key", sourceKey), slog.Any("error", err))
return nil
}
if sourceKey == attachment.StagingKey {
if err := w.repo.MarkStagingDeleted(ctx, attachment.ID, sourceKey); err != nil {
w.log.WarnContext(ctx, "clear staging projection failed", slog.Any("error", err))
}
}
}
return nil
}
func (w *Worker) spoolSource(ctx context.Context, key string) (*os.File, int64, string, error) {
body, _, err := w.store.Open(ctx, key)
if err != nil {
if errors.Is(err, domain.ErrNotFound) {
return nil, 0, "", permanentError{code: "source_missing", err: err}
}
return nil, 0, "", fmt.Errorf("open source: %w", err)
}
defer body.Close()
file, err := os.CreateTemp("", "lab5-worker-*")
if err != nil {
return nil, 0, "", fmt.Errorf("create worker spool: %w", err)
}
cleanup := func() {
_ = file.Close()
_ = os.Remove(file.Name())
}
hasher := sha256.New()
limited := io.LimitReader(body, w.config.MaxUploadBytes+1)
buffer := make([]byte, 64<<10)
size, err := io.CopyBuffer(io.MultiWriter(file, hasher), limited, buffer)
if err != nil {
cleanup()
return nil, 0, "", fmt.Errorf("copy source: %w", err)
}
if size > w.config.MaxUploadBytes {
cleanup()
return nil, 0, "", permanentError{code: "source_too_large", err: fmt.Errorf("source exceeds %d bytes", w.config.MaxUploadBytes)}
}
return file, size, hex.EncodeToString(hasher.Sum(nil)), nil
}
func (w *Worker) fail(ctx context.Context, job domain.Job, err error) broker.Outcome {
if ctx.Err() != nil {
// Parent cancellation means graceful shutdown. Keep the attempt number:
// an operator restart must not consume the processing budget. The job
// context is already cancelled, so use a short detached context for the
// durable requeue.
retryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
failed, retryErr := w.repo.RetryJob(
retryCtx, job.ID, job.Attempt, w.config.Owner, "shutdown",
time.Now().Add(time.Second), true, "shutdown",
)
if retryErr != nil {
return broker.Requeue
}
_ = failed
return broker.Ack
}
code := "processing_failed"
permanent := false
var typed permanentError
if errors.As(err, &typed) {
code = typed.code
permanent = true
}
if errors.Is(err, domain.ErrValidation) || errors.Is(err, domain.ErrUnsupportedMedia) ||
errors.Is(err, domain.ErrTooLarge) || errors.Is(err, domain.ErrConflict) {
permanent = true
if code == "processing_failed" {
code = "invalid_image"
}
}
if errors.Is(err, context.DeadlineExceeded) {
code = "processing_timeout"
permanent = false
}
if permanent {
w.log.WarnContext(ctx, "job permanently failed",
slog.String("job_id", job.ID.String()),
slog.String("error_code", code),
slog.Any("error", err),
)
ok, failErr := w.repo.FailJob(ctx, job.ID, job.Attempt, w.config.Owner, code)
if failErr != nil {
return broker.Requeue
}
if !ok {
return broker.Ack
}
return broker.Drop
}
next := time.Now().Add(w.backoff(job.Attempt))
w.log.WarnContext(ctx, "job retry scheduled",
slog.String("job_id", job.ID.String()),
slog.String("error_code", code),
slog.Time("available_at", next),
slog.Any("error", err),
)
failed, retryErr := w.repo.RetryJob(
ctx, job.ID, job.Attempt, w.config.Owner, code, next, false, "retry",
)
if retryErr != nil {
return broker.Requeue
}
if failed {
return broker.Drop
}
return broker.Ack
}
func (w *Worker) backoff(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
capDelay := w.config.RetryCap
base := w.config.RetryBase
for range attempt - 1 {
base *= 2
if base >= capDelay {
base = capDelay
break
}
}
if base <= 0 {
return time.Second
}
return time.Duration(rand.Int64N(int64(base))) + time.Millisecond
}
type permanentError struct {
code string
err error
}
func (e permanentError) Error() string { return e.code + ": " + e.err.Error() }
func (e permanentError) Unwrap() error { return e.err }