feat(lab5): time pipeline stages and centralize variant constants

This commit is contained in:
Mikhail Verkovykh
2026-11-20 19:05:27 +03:00
parent 1ce93185af
commit d0f533b371
5 changed files with 121 additions and 36 deletions
+11 -1
View File
@@ -22,6 +22,16 @@ const (
// the slack between the global HTTP body limit and the upload limit, so a
// smaller MAX_UPLOAD_BYTES cannot strangle JSON requests.
MaxJSONBodyBytes = 1 * 1024 * 1024
// Image variant parameters. The validator and the worker must agree on the
// contract dimensions, and the experiment records these values, so they
// have exactly one definition.
ThumbnailMax = 256
CropSize = 128
OptimizedMax = 1280
JPEGQuality = 85
WebPQuality = 82
WebPMethod = 4
)
type Config struct {
@@ -63,7 +73,7 @@ type Config struct {
ReconcileGrace time.Duration
ReconcileInterval time.Duration
// Lab5 asynchronous pipeline. BrokerURL is required by the API (outbox
// BrokerURL is required by the API (outbox
// publisher) and the worker (consumer), but not by the reconciler.
BrokerURL string
WorkerOnly bool
+66 -30
View File
@@ -1,4 +1,4 @@
// Package pipeline turns one validated original image into the required lab5
// Package pipeline turns one validated original image into the required
// variants. It is deterministic and side-effect free so a broker redelivery can
// simply overwrite the same keys with the same bytes.
package pipeline
@@ -15,23 +15,16 @@ import (
"image/jpeg"
_ "image/png"
"os"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
"github.com/gen2brain/webp"
"golang.org/x/image/draw"
"golang.org/x/sync/semaphore"
)
const (
ThumbnailMax = 256
CropSize = 128
OptimizedMax = 1280
// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is
// copied byte-for-byte and keeps its metadata and checksum.
JPEGQuality = 85
WebPQuality = 82
WebPMethod = 4
)
// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is
// copied byte-for-byte and keeps its metadata and checksum.
type Processor struct {
maxPixels int64
@@ -83,29 +76,72 @@ func transient(code string, err error) *Failure {
return &Failure{Permanent: false, Code: code, Err: err}
}
// Stats separates decode, pixel scaling and the two encoders. A single
// processing_ms number cannot answer whether a large image costs more because
// of resampling or because of JPEG/WebP compression.
type Stats struct {
Decode time.Duration
Scale time.Duration
JPEGEncode time.Duration
WebPEncode time.Duration
}
func (s Stats) Total() time.Duration {
return s.Decode + s.Scale + s.JPEGEncode + s.WebPEncode
}
// Process decodes the file once and returns all variants in memory. Worker
// concurrency and IMAGE_DECODE_MEMORY_MB bound the number of simultaneous
// decoded images, which is the memory-heavy part of the experiment.
func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, error) {
// decoded images, which is the memory-heavy part of the pipeline.
func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, Stats, error) {
var stats Stats
started := time.Now()
img, err := p.decode(ctx, file)
stats.Decode = time.Since(started)
if err != nil {
return nil, err
return nil, stats, err
}
thumbnail, err := renderJPEG("thumbnail", fitOnWhite(img, ThumbnailMax, ThumbnailMax))
started = time.Now()
thumbnailPixels := fitOnWhite(img, config.ThumbnailMax, config.ThumbnailMax)
stats.Scale += time.Since(started)
thumbnail, encode, err := renderJPEGTimed("thumbnail", thumbnailPixels)
stats.JPEGEncode += encode
if err != nil {
return nil, permanent("processing_failed", err)
}
crop, err := renderJPEG("crop", centerSquare(img, CropSize))
if err != nil {
return nil, permanent("processing_failed", err)
}
optimized, err := renderWebP(fitOnWhite(img, OptimizedMax, OptimizedMax))
if err != nil {
return nil, permanent("processing_failed", err)
return nil, stats, permanent("processing_failed", err)
}
return []Encoded{thumbnail, crop, optimized}, nil
started = time.Now()
cropPixels := centerSquare(img, config.CropSize)
stats.Scale += time.Since(started)
crop, encode, err := renderJPEGTimed("crop", cropPixels)
stats.JPEGEncode += encode
if err != nil {
return nil, stats, permanent("processing_failed", err)
}
started = time.Now()
optimizedPixels := fitOnWhite(img, config.OptimizedMax, config.OptimizedMax)
stats.Scale += time.Since(started)
optimized, encode, err := renderWebPTimed(optimizedPixels)
stats.WebPEncode += encode
if err != nil {
return nil, stats, permanent("processing_failed", err)
}
return []Encoded{thumbnail, crop, optimized}, stats, nil
}
func renderJPEGTimed(name string, img image.Image) (Encoded, time.Duration, error) {
started := time.Now()
encoded, err := renderJPEG(name, img)
return encoded, time.Since(started), err
}
func renderWebPTimed(img image.Image) (Encoded, time.Duration, error) {
started := time.Now()
encoded, err := renderWebP(img)
return encoded, time.Since(started), err
}
func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, error) {
@@ -127,7 +163,7 @@ func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, err
return nil, permanent("invalid_image", errors.New("image exceeds MAX_IMAGE_PIXELS"))
}
// AGENT: the semaphore is acquired before Decode, not after. A decoder
// The semaphore is acquired before Decode, not after. A decoder
// allocates roughly width*height*4 bytes; without this gate several 40 MP
// files can OOM the worker even with a small HTTP concurrency.
weight := pixels * 4
@@ -176,7 +212,7 @@ func fitOnWhite(src image.Image, maxWidth, maxHeight int) *image.RGBA {
return dst
}
// centerSquare always produces CropSize x CropSize. Unlike thumbnails, the
// centerSquare always produces config.CropSize x config.CropSize. Unlike thumbnails, the
// contract allows upscaling tiny sources for the crop variant.
func centerSquare(src image.Image, size int) *image.RGBA {
bounds := src.Bounds()
@@ -216,7 +252,7 @@ func fitSize(width, height, maxWidth, maxHeight int) (int, int) {
func renderJPEG(name string, img image.Image) (Encoded, error) {
var buf bytes.Buffer
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil {
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: config.JPEGQuality}); err != nil {
return Encoded{}, err
}
bounds := img.Bounds()
@@ -232,7 +268,7 @@ func renderJPEG(name string, img image.Image) (Encoded, error) {
func renderWebP(img image.Image) (Encoded, error) {
var buf bytes.Buffer
if err := webp.Encode(&buf, img, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil {
if err := webp.Encode(&buf, img, webp.Options{Quality: config.WebPQuality, Method: config.WebPMethod}); err != nil {
return Encoded{}, err
}
bounds := img.Bounds()
+20 -3
View File
@@ -34,9 +34,26 @@ func TestFitSizePreservesAspectAndNeverUpscales(t *testing.T) {
}
}
func TestProcessReportsTiming(t *testing.T) {
file := pngFile(t, 320, 200, false)
_, stats, err := New(40_000_000, 256<<20).Process(context.Background(), file)
if err != nil {
t.Fatalf("Process: %v", err)
}
if stats.Decode <= 0 || stats.Scale <= 0 || stats.JPEGEncode <= 0 || stats.WebPEncode <= 0 {
t.Fatalf("incomplete timing stats: %+v", stats)
}
if stats.Total() != stats.Decode+stats.Scale+stats.JPEGEncode+stats.WebPEncode {
t.Fatalf("total %v does not match its parts: %+v", stats.Total(), stats)
}
if stats.Total() <= 0 {
t.Fatalf("total timing must be positive: %+v", stats)
}
}
func TestProcessProducesContractVariants(t *testing.T) {
file := pngFile(t, 320, 200, false)
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file)
if err != nil {
t.Fatalf("Process: %v", err)
}
@@ -73,7 +90,7 @@ func TestProcessProducesContractVariants(t *testing.T) {
func TestThumbnailCompositesAlphaOnWhite(t *testing.T) {
file := pngFile(t, 64, 64, true)
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file)
if err != nil {
t.Fatalf("Process: %v", err)
}
@@ -92,7 +109,7 @@ func TestEXIFOrientationRotatesBeforeResize(t *testing.T) {
if got := jpegOrientation(file); got != 6 {
t.Fatalf("jpegOrientation = %d, want 6", got)
}
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file)
if err != nil {
t.Fatalf("Process: %v", err)
}
+12
View File
@@ -12,6 +12,18 @@ import (
"github.com/jackc/pgx/v5"
)
// Job state machine. Every transition is a compare-and-set on
// (job_id, attempt, status), so broker duplicates and stale workers are
// harmless.
//
// queued --ClaimJob--> processing --CompleteJob--> ready
// |
// +--ScheduleRetry--> queued (future available_at)
// +--FailJob--------> failed
// +--attempt >= max-> failed/retry_exhausted
// processing with an expired lease --RecoverExpiredLeases--> queued/failed
// failed --ReplayFailed--> queued (attempt=0, replay_count+1)
//
// ClaimResult tells the consumer whether this delivery owns the work.
type ClaimResult struct {
Job model.JobRecord
+12 -2
View File
@@ -124,7 +124,6 @@ func (w *Worker) Handle(ctx context.Context, message model.Job) error {
return w.handleFailure(ctx, claim, err, log)
}
log.Info("attachment ready")
if w.cfg.CrashAfterEffect {
// Deterministic fault injection for the "written but not acked" window.
// The unacked delivery is redelivered after the connection dies.
@@ -191,7 +190,7 @@ func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")}
}
encoded, err := w.pipeline.Process(ctx, file)
encoded, stats, err := w.pipeline.Process(ctx, file)
if err != nil {
return err
}
@@ -229,6 +228,17 @@ func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog
return nil
}
// The timing line is the experiment's source for compression overhead. It
// is emitted only after the ready snapshot commits, so a redelivered job
// cannot produce two successful samples.
log.Info("attachment ready",
"decode_us", stats.Decode.Microseconds(),
"scale_us", stats.Scale.Microseconds(),
"jpeg_encode_us", stats.JPEGEncode.Microseconds(),
"webp_encode_us", stats.WebPEncode.Microseconds(),
"pipeline_us", stats.Total().Microseconds(),
)
// Staging is removed only after ready commits. A crash before this delete
// leaves an object the reconciler will clean after retention.
if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {