forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
@@ -0,0 +1,298 @@
|
||||
// Package worker runs the asynchronous attachment pipeline. It consumes one
|
||||
// logical job, reads trusted metadata from PostgreSQL, loads the staged object,
|
||||
// writes the original and variants, commits ready atomically, and only then
|
||||
// lets the broker ack the message.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/pipeline"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
cfg config.Config
|
||||
store *store.Store
|
||||
objects *objectstore.Client
|
||||
pipeline *pipeline.Processor
|
||||
logger *slog.Logger
|
||||
faultOnce atomic.Bool
|
||||
}
|
||||
|
||||
func New(cfg config.Config, st *store.Store, objects *objectstore.Client, logger *slog.Logger) *Worker {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
w := &Worker{
|
||||
cfg: cfg,
|
||||
store: st,
|
||||
objects: objects,
|
||||
pipeline: pipeline.New(cfg.MaxImagePixels, cfg.ImageDecodeBytes),
|
||||
logger: logger,
|
||||
}
|
||||
w.faultOnce.Store(true)
|
||||
return w
|
||||
}
|
||||
|
||||
// StartLeaseSweeper recovers a worker that died after claiming a job. Without
|
||||
// this, the database would show "processing" forever and no message would be
|
||||
// left to redeliver.
|
||||
func (w *Worker) StartLeaseSweeper(ctx context.Context) {
|
||||
ticker := time.NewTicker(w.cfg.LeaseSweep)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
count, err := w.store.RecoverExpiredLeases(ctx)
|
||||
if err != nil {
|
||||
w.logger.Error("lease sweep failed", "err", err)
|
||||
} else if count > 0 {
|
||||
w.logger.Warn("recovered expired job leases", "count", count)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle is the broker callback. Returning nil means "durable effect or
|
||||
// deliberate no-op; ack". ErrRequeue means the database itself was unavailable.
|
||||
func (w *Worker) Handle(ctx context.Context, message model.Job) error {
|
||||
claim, err := w.store.ClaimJob(ctx, message.JobID, w.cfg.JobLease)
|
||||
if err != nil {
|
||||
if common.IsCode(err, common.ErrorCodeNotFound) {
|
||||
w.logger.Warn("job no longer exists; acking", "job_id", message.JobID)
|
||||
return nil
|
||||
}
|
||||
w.logger.Error("claim job failed", "job_id", message.JobID, "err", err)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
if claim.Terminal || claim.Busy {
|
||||
w.logger.Info("job is already finished or owned",
|
||||
"job_id", message.JobID, "terminal", claim.Terminal, "busy", claim.Busy,
|
||||
"attempt", claim.Job.Attempt)
|
||||
return nil
|
||||
}
|
||||
if claim.Attachment.ID != message.AttachmentID {
|
||||
w.logger.Error("message attachment mismatch; acking poison message",
|
||||
"job_id", message.JobID, "message_attachment", message.AttachmentID,
|
||||
"database_attachment", claim.Attachment.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
log := w.logger.With(
|
||||
"job_id", claim.Job.JobID,
|
||||
"attachment_id", claim.Attachment.ID,
|
||||
"attempt", claim.Job.Attempt,
|
||||
"image_bytes", claim.Attachment.SizeBytes,
|
||||
)
|
||||
if w.cfg.CrashAfterClaim {
|
||||
// Deterministic fault injection for the "claimed but never finished"
|
||||
// window. The lease sweeper requeues the durable job after it expires;
|
||||
// it does not depend on the broker redelivery still being around.
|
||||
w.logger.Error("fault injection: exiting after claim before processing",
|
||||
"job_id", claim.Job.JobID, "lease_until", claim.Job.LeaseUntil)
|
||||
os.Exit(10)
|
||||
}
|
||||
|
||||
log.Info("processing attachment")
|
||||
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err = w.process(jobCtx, claim, log); err != nil {
|
||||
if jobCtx.Err() != nil && errors.Is(jobCtx.Err(), context.DeadlineExceeded) {
|
||||
err = &pipeline.Failure{Permanent: false, Code: "processing_timeout", Err: err}
|
||||
}
|
||||
return w.handleFailure(ctx, claim, err, log)
|
||||
}
|
||||
|
||||
log.Info("attachment ready")
|
||||
if w.cfg.CrashAfterEffect {
|
||||
// Deterministic fault injection for the "written but not acked" window.
|
||||
// The unacked delivery is redelivered after the connection dies.
|
||||
w.logger.Error("fault injection: exiting after durable effect before ack",
|
||||
"job_id", claim.Job.JobID)
|
||||
os.Exit(9)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog.Logger) error {
|
||||
if mode := w.cfg.WorkerFaultMode; mode != "none" && w.faultOnce.CompareAndSwap(true, false) {
|
||||
if mode == "transient-once" {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed",
|
||||
Err: errors.New("injected transient failure")}
|
||||
}
|
||||
return &pipeline.Failure{Permanent: true, Code: "invalid_image",
|
||||
Err: errors.New("injected permanent failure")}
|
||||
}
|
||||
|
||||
att := claim.Attachment
|
||||
if att.StagingKey == "" {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_missing",
|
||||
Err: errors.New("attachment has no staging key")}
|
||||
}
|
||||
|
||||
reader, size, err := w.objects.Get(ctx, att.StagingKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, objectstore.ErrNotFound) {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_missing", Err: err}
|
||||
}
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
defer reader.Close()
|
||||
if size != att.SizeBytes {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt",
|
||||
Err: fmt.Errorf("staged size %d != metadata %d", size, att.SizeBytes)}
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(w.cfg.WorkTempDir, 0o700); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
file, err := os.CreateTemp(w.cfg.WorkTempDir, "job-*.part")
|
||||
if err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
path := file.Name()
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(path)
|
||||
}()
|
||||
|
||||
hasher := sha256.New()
|
||||
// The same 10 MiB contract limit protects the worker from a corrupt row
|
||||
// pointing at an unexpectedly large object.
|
||||
written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, w.cfg.MaxUploadBytes+1), make([]byte, 32*1024))
|
||||
if err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
if written > w.cfg.MaxUploadBytes {
|
||||
return &pipeline.Failure{Permanent: true, Code: "invalid_image", Err: errors.New("staged object too large")}
|
||||
}
|
||||
if written != att.SizeBytes || hex.EncodeToString(hasher.Sum(nil)) != att.SHA256 {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")}
|
||||
}
|
||||
|
||||
encoded, err := w.pipeline.Process(ctx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Original bytes are copied unchanged from staging to the final key.
|
||||
if _, err = file.Seek(0, 0); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
if err = w.objects.Put(ctx, att.ObjectKey, file, att.SizeBytes, att.ContentType); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
|
||||
variants := make([]model.Variant, 0, len(encoded))
|
||||
for _, item := range encoded {
|
||||
key := store.VariantObjectKey(att.ObjectKey, item.Name)
|
||||
if err = w.objects.Put(ctx, key, bytes.NewReader(item.Bytes), int64(len(item.Bytes)), item.ContentType); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
variants = append(variants, model.Variant{
|
||||
Name: item.Name,
|
||||
ContentType: item.ContentType,
|
||||
SizeBytes: int64(len(item.Bytes)),
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
SHA256: item.SHA256,
|
||||
})
|
||||
}
|
||||
|
||||
committed, err := w.store.CompleteJob(ctx, claim.Job.JobID, claim.Job.Attempt, variants)
|
||||
if err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
if !committed {
|
||||
log.Warn("ready snapshot was superseded; leaving newer state alone")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Staging is removed only after ready commits. A crash before this delete
|
||||
// leaves an object the reconciler will clean after retention.
|
||||
if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
log.Warn("staging delete failed; reconciler will retry", "err", err)
|
||||
return nil
|
||||
}
|
||||
if err = w.store.ClearStagingKey(ctx, att.ID); err != nil {
|
||||
log.Warn("clear staging key failed; reconciler will retry", "err", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) handleFailure(ctx context.Context, claim store.ClaimResult, err error, log *slog.Logger) error {
|
||||
var failure *pipeline.Failure
|
||||
if !errors.As(err, &failure) {
|
||||
failure = &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
|
||||
if failure.Permanent {
|
||||
log.Error("permanent job failure", "code", failure.Code, "err", failure.Err)
|
||||
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code); failErr != nil {
|
||||
log.Error("record permanent failure failed; requeueing", "err", failErr)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if claim.Job.Attempt >= claim.Job.MaxAttempts {
|
||||
log.Error("retry budget exhausted", "code", failure.Code, "err", failure.Err)
|
||||
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, "retry_exhausted"); failErr != nil {
|
||||
log.Error("record exhausted retry failed; requeueing", "err", failErr)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
delay := w.backoff(claim.Job.Attempt)
|
||||
log.Warn("transient job failure; scheduling retry",
|
||||
"code", failure.Code, "err", failure.Err, "retry_in", delay)
|
||||
scheduled, retryErr := w.store.ScheduleRetry(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code, delay)
|
||||
if retryErr != nil {
|
||||
log.Error("schedule retry failed; requeueing", "err", retryErr)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
if !scheduled {
|
||||
// This attempt no longer owns the row (for example a lease sweep or a
|
||||
// replay changed it). Requeue rather than ack a job whose future work
|
||||
// was not durably recorded.
|
||||
log.Warn("retry scheduling lost ownership; requeueing")
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// backoff is exponential with +/-50% jitter, capped by JOB_RETRY_MAX_SECONDS.
|
||||
// Jitter prevents many retries from hitting a recovered dependency together.
|
||||
func (w *Worker) backoff(attempt int) time.Duration {
|
||||
delay := w.cfg.RetryBase
|
||||
for i := 1; i < attempt && delay < w.cfg.RetryMax; i++ {
|
||||
delay *= 2
|
||||
}
|
||||
if delay > w.cfg.RetryMax {
|
||||
delay = w.cfg.RetryMax
|
||||
}
|
||||
jitter := 0.5 + rand.Float64() // 0.5 .. 1.5
|
||||
return time.Duration(float64(delay) * jitter)
|
||||
}
|
||||
Reference in New Issue
Block a user