diff --git a/app/internal/config/config.go b/app/internal/config/config.go index 3b71d110..bde7501d 100644 --- a/app/internal/config/config.go +++ b/app/internal/config/config.go @@ -22,6 +22,16 @@ const ( // the slack between the global HTTP body limit and the upload limit, so a // smaller MAX_UPLOAD_BYTES cannot strangle JSON requests. MaxJSONBodyBytes = 1 * 1024 * 1024 + + // Image variant parameters. The validator and the worker must agree on the + // contract dimensions, and the experiment records these values, so they + // have exactly one definition. + ThumbnailMax = 256 + CropSize = 128 + OptimizedMax = 1280 + JPEGQuality = 85 + WebPQuality = 82 + WebPMethod = 4 ) type Config struct { @@ -63,7 +73,7 @@ type Config struct { ReconcileGrace time.Duration ReconcileInterval time.Duration - // Lab5 asynchronous pipeline. BrokerURL is required by the API (outbox + // BrokerURL is required by the API (outbox // publisher) and the worker (consumer), but not by the reconciler. BrokerURL string WorkerOnly bool diff --git a/app/internal/pipeline/image.go b/app/internal/pipeline/image.go index 1713a42c..92256c02 100644 --- a/app/internal/pipeline/image.go +++ b/app/internal/pipeline/image.go @@ -1,4 +1,4 @@ -// Package pipeline turns one validated original image into the required lab5 +// Package pipeline turns one validated original image into the required // variants. It is deterministic and side-effect free so a broker redelivery can // simply overwrite the same keys with the same bytes. package pipeline @@ -15,23 +15,16 @@ import ( "image/jpeg" _ "image/png" "os" + "time" + "gitflic.ru/project/bulla-max/mtusi2026lab/internal/config" "github.com/gen2brain/webp" "golang.org/x/image/draw" "golang.org/x/sync/semaphore" ) -const ( - ThumbnailMax = 256 - CropSize = 128 - OptimizedMax = 1280 - - // Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is - // copied byte-for-byte and keeps its metadata and checksum. - JPEGQuality = 85 - WebPQuality = 82 - WebPMethod = 4 -) +// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is +// copied byte-for-byte and keeps its metadata and checksum. type Processor struct { maxPixels int64 @@ -83,29 +76,72 @@ func transient(code string, err error) *Failure { return &Failure{Permanent: false, Code: code, Err: err} } +// Stats separates decode, pixel scaling and the two encoders. A single +// processing_ms number cannot answer whether a large image costs more because +// of resampling or because of JPEG/WebP compression. +type Stats struct { + Decode time.Duration + Scale time.Duration + JPEGEncode time.Duration + WebPEncode time.Duration +} + +func (s Stats) Total() time.Duration { + return s.Decode + s.Scale + s.JPEGEncode + s.WebPEncode +} + // Process decodes the file once and returns all variants in memory. Worker // concurrency and IMAGE_DECODE_MEMORY_MB bound the number of simultaneous -// decoded images, which is the memory-heavy part of the experiment. -func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, error) { +// decoded images, which is the memory-heavy part of the pipeline. +func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, Stats, error) { + var stats Stats + started := time.Now() img, err := p.decode(ctx, file) + stats.Decode = time.Since(started) if err != nil { - return nil, err + return nil, stats, err } - thumbnail, err := renderJPEG("thumbnail", fitOnWhite(img, ThumbnailMax, ThumbnailMax)) + started = time.Now() + thumbnailPixels := fitOnWhite(img, config.ThumbnailMax, config.ThumbnailMax) + stats.Scale += time.Since(started) + thumbnail, encode, err := renderJPEGTimed("thumbnail", thumbnailPixels) + stats.JPEGEncode += encode if err != nil { - return nil, permanent("processing_failed", err) - } - crop, err := renderJPEG("crop", centerSquare(img, CropSize)) - if err != nil { - return nil, permanent("processing_failed", err) - } - optimized, err := renderWebP(fitOnWhite(img, OptimizedMax, OptimizedMax)) - if err != nil { - return nil, permanent("processing_failed", err) + return nil, stats, permanent("processing_failed", err) } - return []Encoded{thumbnail, crop, optimized}, nil + started = time.Now() + cropPixels := centerSquare(img, config.CropSize) + stats.Scale += time.Since(started) + crop, encode, err := renderJPEGTimed("crop", cropPixels) + stats.JPEGEncode += encode + if err != nil { + return nil, stats, permanent("processing_failed", err) + } + + started = time.Now() + optimizedPixels := fitOnWhite(img, config.OptimizedMax, config.OptimizedMax) + stats.Scale += time.Since(started) + optimized, encode, err := renderWebPTimed(optimizedPixels) + stats.WebPEncode += encode + if err != nil { + return nil, stats, permanent("processing_failed", err) + } + + return []Encoded{thumbnail, crop, optimized}, stats, nil +} + +func renderJPEGTimed(name string, img image.Image) (Encoded, time.Duration, error) { + started := time.Now() + encoded, err := renderJPEG(name, img) + return encoded, time.Since(started), err +} + +func renderWebPTimed(img image.Image) (Encoded, time.Duration, error) { + started := time.Now() + encoded, err := renderWebP(img) + return encoded, time.Since(started), err } func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, error) { @@ -127,7 +163,7 @@ func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, err return nil, permanent("invalid_image", errors.New("image exceeds MAX_IMAGE_PIXELS")) } - // AGENT: the semaphore is acquired before Decode, not after. A decoder + // The semaphore is acquired before Decode, not after. A decoder // allocates roughly width*height*4 bytes; without this gate several 40 MP // files can OOM the worker even with a small HTTP concurrency. weight := pixels * 4 @@ -176,7 +212,7 @@ func fitOnWhite(src image.Image, maxWidth, maxHeight int) *image.RGBA { return dst } -// centerSquare always produces CropSize x CropSize. Unlike thumbnails, the +// centerSquare always produces config.CropSize x config.CropSize. Unlike thumbnails, the // contract allows upscaling tiny sources for the crop variant. func centerSquare(src image.Image, size int) *image.RGBA { bounds := src.Bounds() @@ -216,7 +252,7 @@ func fitSize(width, height, maxWidth, maxHeight int) (int, int) { func renderJPEG(name string, img image.Image) (Encoded, error) { var buf bytes.Buffer - if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil { + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: config.JPEGQuality}); err != nil { return Encoded{}, err } bounds := img.Bounds() @@ -232,7 +268,7 @@ func renderJPEG(name string, img image.Image) (Encoded, error) { func renderWebP(img image.Image) (Encoded, error) { var buf bytes.Buffer - if err := webp.Encode(&buf, img, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil { + if err := webp.Encode(&buf, img, webp.Options{Quality: config.WebPQuality, Method: config.WebPMethod}); err != nil { return Encoded{}, err } bounds := img.Bounds() diff --git a/app/internal/pipeline/image_test.go b/app/internal/pipeline/image_test.go index 2a726ec7..ffc9e98e 100644 --- a/app/internal/pipeline/image_test.go +++ b/app/internal/pipeline/image_test.go @@ -34,9 +34,26 @@ func TestFitSizePreservesAspectAndNeverUpscales(t *testing.T) { } } +func TestProcessReportsTiming(t *testing.T) { + file := pngFile(t, 320, 200, false) + _, stats, err := New(40_000_000, 256<<20).Process(context.Background(), file) + if err != nil { + t.Fatalf("Process: %v", err) + } + if stats.Decode <= 0 || stats.Scale <= 0 || stats.JPEGEncode <= 0 || stats.WebPEncode <= 0 { + t.Fatalf("incomplete timing stats: %+v", stats) + } + if stats.Total() != stats.Decode+stats.Scale+stats.JPEGEncode+stats.WebPEncode { + t.Fatalf("total %v does not match its parts: %+v", stats.Total(), stats) + } + if stats.Total() <= 0 { + t.Fatalf("total timing must be positive: %+v", stats) + } +} + func TestProcessProducesContractVariants(t *testing.T) { file := pngFile(t, 320, 200, false) - variants, err := New(40_000_000, 256<<20).Process(context.Background(), file) + variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file) if err != nil { t.Fatalf("Process: %v", err) } @@ -73,7 +90,7 @@ func TestProcessProducesContractVariants(t *testing.T) { func TestThumbnailCompositesAlphaOnWhite(t *testing.T) { file := pngFile(t, 64, 64, true) - variants, err := New(40_000_000, 256<<20).Process(context.Background(), file) + variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file) if err != nil { t.Fatalf("Process: %v", err) } @@ -92,7 +109,7 @@ func TestEXIFOrientationRotatesBeforeResize(t *testing.T) { if got := jpegOrientation(file); got != 6 { t.Fatalf("jpegOrientation = %d, want 6", got) } - variants, err := New(40_000_000, 256<<20).Process(context.Background(), file) + variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file) if err != nil { t.Fatalf("Process: %v", err) } diff --git a/app/internal/store/jobs.go b/app/internal/store/jobs.go index f0224253..5194767e 100644 --- a/app/internal/store/jobs.go +++ b/app/internal/store/jobs.go @@ -12,6 +12,18 @@ import ( "github.com/jackc/pgx/v5" ) +// Job state machine. Every transition is a compare-and-set on +// (job_id, attempt, status), so broker duplicates and stale workers are +// harmless. +// +// queued --ClaimJob--> processing --CompleteJob--> ready +// | +// +--ScheduleRetry--> queued (future available_at) +// +--FailJob--------> failed +// +--attempt >= max-> failed/retry_exhausted +// processing with an expired lease --RecoverExpiredLeases--> queued/failed +// failed --ReplayFailed--> queued (attempt=0, replay_count+1) +// // ClaimResult tells the consumer whether this delivery owns the work. type ClaimResult struct { Job model.JobRecord diff --git a/app/internal/worker/worker.go b/app/internal/worker/worker.go index f15dea05..50b3354f 100644 --- a/app/internal/worker/worker.go +++ b/app/internal/worker/worker.go @@ -124,7 +124,6 @@ func (w *Worker) Handle(ctx context.Context, message model.Job) error { return w.handleFailure(ctx, claim, err, log) } - log.Info("attachment ready") if w.cfg.CrashAfterEffect { // Deterministic fault injection for the "written but not acked" window. // The unacked delivery is redelivered after the connection dies. @@ -191,7 +190,7 @@ func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")} } - encoded, err := w.pipeline.Process(ctx, file) + encoded, stats, err := w.pipeline.Process(ctx, file) if err != nil { return err } @@ -229,6 +228,17 @@ func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog return nil } + // The timing line is the experiment's source for compression overhead. It + // is emitted only after the ready snapshot commits, so a redelivered job + // cannot produce two successful samples. + log.Info("attachment ready", + "decode_us", stats.Decode.Microseconds(), + "scale_us", stats.Scale.Microseconds(), + "jpeg_encode_us", stats.JPEGEncode.Microseconds(), + "webp_encode_us", stats.WebPEncode.Microseconds(), + "pipeline_us", stats.Total().Microseconds(), + ) + // Staging is removed only after ready commits. A crash before this delete // leaves an object the reconciler will clean after retention. if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {