Files

309 lines
11 KiB
Go

// Package worker runs the asynchronous attachment pipeline. It consumes one
// logical job, reads trusted metadata from PostgreSQL, loads the staged object,
// writes the original and variants, commits ready atomically, and only then
// lets the broker ack the message.
package worker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"os"
"sync/atomic"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/pipeline"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
)
type Worker struct {
cfg config.Config
store *store.Store
objects *objectstore.Client
pipeline *pipeline.Processor
logger *slog.Logger
faultOnce atomic.Bool
}
func New(cfg config.Config, st *store.Store, objects *objectstore.Client, logger *slog.Logger) *Worker {
if logger == nil {
logger = slog.Default()
}
w := &Worker{
cfg: cfg,
store: st,
objects: objects,
pipeline: pipeline.New(cfg.MaxImagePixels, cfg.ImageDecodeBytes),
logger: logger,
}
w.faultOnce.Store(true)
return w
}
// StartLeaseSweeper recovers a worker that died after claiming a job. Without
// this, the database would show "processing" forever and no message would be
// left to redeliver.
func (w *Worker) StartLeaseSweeper(ctx context.Context) {
ticker := time.NewTicker(w.cfg.LeaseSweep)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
count, err := w.store.RecoverExpiredLeases(ctx)
if err != nil {
w.logger.Error("lease sweep failed", "err", err)
} else if count > 0 {
w.logger.Warn("recovered expired job leases", "count", count)
}
}
}
}
// Handle is the broker callback. Returning nil means "durable effect or
// deliberate no-op; ack". ErrRequeue means the database itself was unavailable.
func (w *Worker) Handle(ctx context.Context, message model.Job) error {
claim, err := w.store.ClaimJob(ctx, message.JobID, w.cfg.JobLease)
if err != nil {
if common.IsCode(err, common.ErrorCodeNotFound) {
w.logger.Warn("job no longer exists; acking", "job_id", message.JobID)
return nil
}
w.logger.Error("claim job failed", "job_id", message.JobID, "err", err)
return broker.ErrRequeue
}
if claim.Terminal || claim.Busy {
w.logger.Info("job is already finished or owned",
"job_id", message.JobID, "terminal", claim.Terminal, "busy", claim.Busy,
"attempt", claim.Job.Attempt)
return nil
}
if claim.Attachment.ID != message.AttachmentID {
w.logger.Error("message attachment mismatch; acking poison message",
"job_id", message.JobID, "message_attachment", message.AttachmentID,
"database_attachment", claim.Attachment.ID)
return nil
}
log := w.logger.With(
"job_id", claim.Job.JobID,
"attachment_id", claim.Attachment.ID,
"attempt", claim.Job.Attempt,
"image_bytes", claim.Attachment.SizeBytes,
)
if w.cfg.CrashAfterClaim {
// Deterministic fault injection for the "claimed but never finished"
// window. The lease sweeper requeues the durable job after it expires;
// it does not depend on the broker redelivery still being around.
w.logger.Error("fault injection: exiting after claim before processing",
"job_id", claim.Job.JobID, "lease_until", claim.Job.LeaseUntil)
os.Exit(10)
}
log.Info("processing attachment")
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout)
defer cancel()
if err = w.process(jobCtx, claim, log); err != nil {
if jobCtx.Err() != nil && errors.Is(jobCtx.Err(), context.DeadlineExceeded) {
err = &pipeline.Failure{Permanent: false, Code: "processing_timeout", Err: err}
}
return w.handleFailure(ctx, claim, err, log)
}
if w.cfg.CrashAfterEffect {
// Deterministic fault injection for the "written but not acked" window.
// The unacked delivery is redelivered after the connection dies.
w.logger.Error("fault injection: exiting after durable effect before ack",
"job_id", claim.Job.JobID)
os.Exit(9)
}
return nil
}
func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog.Logger) error {
if mode := w.cfg.WorkerFaultMode; mode != "none" && w.faultOnce.CompareAndSwap(true, false) {
if mode == "transient-once" {
return &pipeline.Failure{Permanent: false, Code: "processing_failed",
Err: errors.New("injected transient failure")}
}
return &pipeline.Failure{Permanent: true, Code: "invalid_image",
Err: errors.New("injected permanent failure")}
}
att := claim.Attachment
if att.StagingKey == "" {
return &pipeline.Failure{Permanent: true, Code: "staging_missing",
Err: errors.New("attachment has no staging key")}
}
reader, size, err := w.objects.Get(ctx, att.StagingKey)
if err != nil {
if errors.Is(err, objectstore.ErrNotFound) {
return &pipeline.Failure{Permanent: true, Code: "staging_missing", Err: err}
}
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
defer reader.Close()
if size != att.SizeBytes {
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt",
Err: fmt.Errorf("staged size %d != metadata %d", size, att.SizeBytes)}
}
if err = os.MkdirAll(w.cfg.WorkTempDir, 0o700); err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
file, err := os.CreateTemp(w.cfg.WorkTempDir, "job-*.part")
if err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
path := file.Name()
defer func() {
_ = file.Close()
_ = os.Remove(path)
}()
hasher := sha256.New()
// The same 10 MiB contract limit protects the worker from a corrupt row
// pointing at an unexpectedly large object.
written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, w.cfg.MaxUploadBytes+1), make([]byte, 32*1024))
if err != nil {
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
if written > w.cfg.MaxUploadBytes {
return &pipeline.Failure{Permanent: true, Code: "invalid_image", Err: errors.New("staged object too large")}
}
if written != att.SizeBytes || hex.EncodeToString(hasher.Sum(nil)) != att.SHA256 {
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")}
}
encoded, stats, err := w.pipeline.Process(ctx, file)
if err != nil {
return err
}
// Original bytes are copied unchanged from staging to the final key.
if _, err = file.Seek(0, 0); err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
if err = w.objects.Put(ctx, att.ObjectKey, file, att.SizeBytes, att.ContentType); err != nil {
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
variants := make([]model.Variant, 0, len(encoded))
for _, item := range encoded {
key := store.VariantObjectKey(att.ObjectKey, item.Name)
if err = w.objects.Put(ctx, key, bytes.NewReader(item.Bytes), int64(len(item.Bytes)), item.ContentType); err != nil {
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
variants = append(variants, model.Variant{
Name: item.Name,
ContentType: item.ContentType,
SizeBytes: int64(len(item.Bytes)),
Width: item.Width,
Height: item.Height,
SHA256: item.SHA256,
})
}
committed, err := w.store.CompleteJob(ctx, claim.Job.JobID, claim.Job.Attempt, variants)
if err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
if !committed {
log.Warn("ready snapshot was superseded; leaving newer state alone")
return nil
}
// The timing line is the experiment's source for compression overhead. It
// is emitted only after the ready snapshot commits, so a redelivered job
// cannot produce two successful samples.
log.Info("attachment ready",
"decode_us", stats.Decode.Microseconds(),
"scale_us", stats.Scale.Microseconds(),
"jpeg_encode_us", stats.JPEGEncode.Microseconds(),
"webp_encode_us", stats.WebPEncode.Microseconds(),
"pipeline_us", stats.Total().Microseconds(),
)
// Staging is removed only after ready commits. A crash before this delete
// leaves an object the reconciler will clean after retention.
if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
log.Warn("staging delete failed; reconciler will retry", "err", err)
return nil
}
if err = w.store.ClearStagingKey(ctx, att.ID); err != nil {
log.Warn("clear staging key failed; reconciler will retry", "err", err)
}
return nil
}
func (w *Worker) handleFailure(ctx context.Context, claim store.ClaimResult, err error, log *slog.Logger) error {
var failure *pipeline.Failure
if !errors.As(err, &failure) {
failure = &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
if failure.Permanent {
log.Error("permanent job failure", "code", failure.Code, "err", failure.Err)
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code); failErr != nil {
log.Error("record permanent failure failed; requeueing", "err", failErr)
return broker.ErrRequeue
}
return nil
}
if claim.Job.Attempt >= claim.Job.MaxAttempts {
log.Error("retry budget exhausted", "code", failure.Code, "err", failure.Err)
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, "retry_exhausted"); failErr != nil {
log.Error("record exhausted retry failed; requeueing", "err", failErr)
return broker.ErrRequeue
}
return nil
}
delay := w.backoff(claim.Job.Attempt)
log.Warn("transient job failure; scheduling retry",
"code", failure.Code, "err", failure.Err, "retry_in", delay)
scheduled, retryErr := w.store.ScheduleRetry(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code, delay)
if retryErr != nil {
log.Error("schedule retry failed; requeueing", "err", retryErr)
return broker.ErrRequeue
}
if !scheduled {
// This attempt no longer owns the row (for example a lease sweep or a
// replay changed it). Requeue rather than ack a job whose future work
// was not durably recorded.
log.Warn("retry scheduling lost ownership; requeueing")
return broker.ErrRequeue
}
return nil
}
// backoff is exponential with +/-50% jitter, capped by JOB_RETRY_MAX_SECONDS.
// Jitter prevents many retries from hitting a recovered dependency together.
func (w *Worker) backoff(attempt int) time.Duration {
delay := w.cfg.RetryBase
for i := 1; i < attempt && delay < w.cfg.RetryMax; i++ {
delay *= 2
}
if delay > w.cfg.RetryMax {
delay = w.cfg.RetryMax
}
jitter := 0.5 + rand.Float64() // 0.5 .. 1.5
return time.Duration(float64(delay) * jitter)
}