// Package worker runs the asynchronous attachment pipeline. It consumes one // logical job, reads trusted metadata from PostgreSQL, loads the staged object, // writes the original and variants, commits ready atomically, and only then // lets the broker ack the message. package worker import ( "bytes" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "log/slog" "math/rand/v2" "os" "sync/atomic" "time" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/common" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/model" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/pipeline" "gitflic.ru/project/bulla-max/mtusi2026lab/internal/store" ) type Worker struct { cfg config.Config store *store.Store objects *objectstore.Client pipeline *pipeline.Processor logger *slog.Logger faultOnce atomic.Bool } func New(cfg config.Config, st *store.Store, objects *objectstore.Client, logger *slog.Logger) *Worker { if logger == nil { logger = slog.Default() } w := &Worker{ cfg: cfg, store: st, objects: objects, pipeline: pipeline.New(cfg.MaxImagePixels, cfg.ImageDecodeBytes), logger: logger, } w.faultOnce.Store(true) return w } // StartLeaseSweeper recovers a worker that died after claiming a job. Without // this, the database would show "processing" forever and no message would be // left to redeliver. func (w *Worker) StartLeaseSweeper(ctx context.Context) { ticker := time.NewTicker(w.cfg.LeaseSweep) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: count, err := w.store.RecoverExpiredLeases(ctx) if err != nil { w.logger.Error("lease sweep failed", "err", err) } else if count > 0 { w.logger.Warn("recovered expired job leases", "count", count) } } } } // Handle is the broker callback. Returning nil means "durable effect or // deliberate no-op; ack". ErrRequeue means the database itself was unavailable. func (w *Worker) Handle(ctx context.Context, message model.Job) error { claim, err := w.store.ClaimJob(ctx, message.JobID, w.cfg.JobLease) if err != nil { if common.IsCode(err, common.ErrorCodeNotFound) { w.logger.Warn("job no longer exists; acking", "job_id", message.JobID) return nil } w.logger.Error("claim job failed", "job_id", message.JobID, "err", err) return broker.ErrRequeue } if claim.Terminal || claim.Busy { w.logger.Info("job is already finished or owned", "job_id", message.JobID, "terminal", claim.Terminal, "busy", claim.Busy, "attempt", claim.Job.Attempt) return nil } if claim.Attachment.ID != message.AttachmentID { w.logger.Error("message attachment mismatch; acking poison message", "job_id", message.JobID, "message_attachment", message.AttachmentID, "database_attachment", claim.Attachment.ID) return nil } log := w.logger.With( "job_id", claim.Job.JobID, "attachment_id", claim.Attachment.ID, "attempt", claim.Job.Attempt, "image_bytes", claim.Attachment.SizeBytes, ) if w.cfg.CrashAfterClaim { // Deterministic fault injection for the "claimed but never finished" // window. The lease sweeper requeues the durable job after it expires; // it does not depend on the broker redelivery still being around. w.logger.Error("fault injection: exiting after claim before processing", "job_id", claim.Job.JobID, "lease_until", claim.Job.LeaseUntil) os.Exit(10) } log.Info("processing attachment") jobCtx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout) defer cancel() if err = w.process(jobCtx, claim, log); err != nil { if jobCtx.Err() != nil && errors.Is(jobCtx.Err(), context.DeadlineExceeded) { err = &pipeline.Failure{Permanent: false, Code: "processing_timeout", Err: err} } return w.handleFailure(ctx, claim, err, log) } if w.cfg.CrashAfterEffect { // Deterministic fault injection for the "written but not acked" window. // The unacked delivery is redelivered after the connection dies. w.logger.Error("fault injection: exiting after durable effect before ack", "job_id", claim.Job.JobID) os.Exit(9) } return nil } func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog.Logger) error { if mode := w.cfg.WorkerFaultMode; mode != "none" && w.faultOnce.CompareAndSwap(true, false) { if mode == "transient-once" { return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: errors.New("injected transient failure")} } return &pipeline.Failure{Permanent: true, Code: "invalid_image", Err: errors.New("injected permanent failure")} } att := claim.Attachment if att.StagingKey == "" { return &pipeline.Failure{Permanent: true, Code: "staging_missing", Err: errors.New("attachment has no staging key")} } reader, size, err := w.objects.Get(ctx, att.StagingKey) if err != nil { if errors.Is(err, objectstore.ErrNotFound) { return &pipeline.Failure{Permanent: true, Code: "staging_missing", Err: err} } return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} } defer reader.Close() if size != att.SizeBytes { return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: fmt.Errorf("staged size %d != metadata %d", size, att.SizeBytes)} } if err = os.MkdirAll(w.cfg.WorkTempDir, 0o700); err != nil { return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} } file, err := os.CreateTemp(w.cfg.WorkTempDir, "job-*.part") if err != nil { return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} } path := file.Name() defer func() { _ = file.Close() _ = os.Remove(path) }() hasher := sha256.New() // The same 10 MiB contract limit protects the worker from a corrupt row // pointing at an unexpectedly large object. written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, w.cfg.MaxUploadBytes+1), make([]byte, 32*1024)) if err != nil { return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} } if written > w.cfg.MaxUploadBytes { return &pipeline.Failure{Permanent: true, Code: "invalid_image", Err: errors.New("staged object too large")} } if written != att.SizeBytes || hex.EncodeToString(hasher.Sum(nil)) != att.SHA256 { return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")} } encoded, stats, err := w.pipeline.Process(ctx, file) if err != nil { return err } // Original bytes are copied unchanged from staging to the final key. if _, err = file.Seek(0, 0); err != nil { return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} } if err = w.objects.Put(ctx, att.ObjectKey, file, att.SizeBytes, att.ContentType); err != nil { return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} } variants := make([]model.Variant, 0, len(encoded)) for _, item := range encoded { key := store.VariantObjectKey(att.ObjectKey, item.Name) if err = w.objects.Put(ctx, key, bytes.NewReader(item.Bytes), int64(len(item.Bytes)), item.ContentType); err != nil { return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err} } variants = append(variants, model.Variant{ Name: item.Name, ContentType: item.ContentType, SizeBytes: int64(len(item.Bytes)), Width: item.Width, Height: item.Height, SHA256: item.SHA256, }) } committed, err := w.store.CompleteJob(ctx, claim.Job.JobID, claim.Job.Attempt, variants) if err != nil { return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} } if !committed { log.Warn("ready snapshot was superseded; leaving newer state alone") return nil } // The timing line is the experiment's source for compression overhead. It // is emitted only after the ready snapshot commits, so a redelivered job // cannot produce two successful samples. log.Info("attachment ready", "decode_us", stats.Decode.Microseconds(), "scale_us", stats.Scale.Microseconds(), "jpeg_encode_us", stats.JPEGEncode.Microseconds(), "webp_encode_us", stats.WebPEncode.Microseconds(), "pipeline_us", stats.Total().Microseconds(), ) // Staging is removed only after ready commits. A crash before this delete // leaves an object the reconciler will clean after retention. if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) { log.Warn("staging delete failed; reconciler will retry", "err", err) return nil } if err = w.store.ClearStagingKey(ctx, att.ID); err != nil { log.Warn("clear staging key failed; reconciler will retry", "err", err) } return nil } func (w *Worker) handleFailure(ctx context.Context, claim store.ClaimResult, err error, log *slog.Logger) error { var failure *pipeline.Failure if !errors.As(err, &failure) { failure = &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err} } if failure.Permanent { log.Error("permanent job failure", "code", failure.Code, "err", failure.Err) if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code); failErr != nil { log.Error("record permanent failure failed; requeueing", "err", failErr) return broker.ErrRequeue } return nil } if claim.Job.Attempt >= claim.Job.MaxAttempts { log.Error("retry budget exhausted", "code", failure.Code, "err", failure.Err) if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, "retry_exhausted"); failErr != nil { log.Error("record exhausted retry failed; requeueing", "err", failErr) return broker.ErrRequeue } return nil } delay := w.backoff(claim.Job.Attempt) log.Warn("transient job failure; scheduling retry", "code", failure.Code, "err", failure.Err, "retry_in", delay) scheduled, retryErr := w.store.ScheduleRetry(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code, delay) if retryErr != nil { log.Error("schedule retry failed; requeueing", "err", retryErr) return broker.ErrRequeue } if !scheduled { // This attempt no longer owns the row (for example a lease sweep or a // replay changed it). Requeue rather than ack a job whose future work // was not durably recorded. log.Warn("retry scheduling lost ownership; requeueing") return broker.ErrRequeue } return nil } // backoff is exponential with +/-50% jitter, capped by JOB_RETRY_MAX_SECONDS. // Jitter prevents many retries from hitting a recovered dependency together. func (w *Worker) backoff(attempt int) time.Duration { delay := w.cfg.RetryBase for i := 1; i < attempt && delay < w.cfg.RetryMax; i++ { delay *= 2 } if delay > w.cfg.RetryMax { delay = w.cfg.RetryMax } jitter := 0.5 + rand.Float64() // 0.5 .. 1.5 return time.Duration(float64(delay) * jitter) }