forked from mtusi2026/mtusi2026lab5
feat(lab5): time pipeline stages and centralize variant constants
This commit is contained in:
@@ -22,6 +22,16 @@ const (
|
||||
// the slack between the global HTTP body limit and the upload limit, so a
|
||||
// smaller MAX_UPLOAD_BYTES cannot strangle JSON requests.
|
||||
MaxJSONBodyBytes = 1 * 1024 * 1024
|
||||
|
||||
// Image variant parameters. The validator and the worker must agree on the
|
||||
// contract dimensions, and the experiment records these values, so they
|
||||
// have exactly one definition.
|
||||
ThumbnailMax = 256
|
||||
CropSize = 128
|
||||
OptimizedMax = 1280
|
||||
JPEGQuality = 85
|
||||
WebPQuality = 82
|
||||
WebPMethod = 4
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -63,7 +73,7 @@ type Config struct {
|
||||
ReconcileGrace time.Duration
|
||||
ReconcileInterval time.Duration
|
||||
|
||||
// Lab5 asynchronous pipeline. BrokerURL is required by the API (outbox
|
||||
// BrokerURL is required by the API (outbox
|
||||
// publisher) and the worker (consumer), but not by the reconciler.
|
||||
BrokerURL string
|
||||
WorkerOnly bool
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Package pipeline turns one validated original image into the required lab5
|
||||
// Package pipeline turns one validated original image into the required
|
||||
// variants. It is deterministic and side-effect free so a broker redelivery can
|
||||
// simply overwrite the same keys with the same bytes.
|
||||
package pipeline
|
||||
@@ -15,23 +15,16 @@ import (
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
|
||||
"github.com/gen2brain/webp"
|
||||
"golang.org/x/image/draw"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
const (
|
||||
ThumbnailMax = 256
|
||||
CropSize = 128
|
||||
OptimizedMax = 1280
|
||||
|
||||
// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is
|
||||
// copied byte-for-byte and keeps its metadata and checksum.
|
||||
JPEGQuality = 85
|
||||
WebPQuality = 82
|
||||
WebPMethod = 4
|
||||
)
|
||||
// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is
|
||||
// copied byte-for-byte and keeps its metadata and checksum.
|
||||
|
||||
type Processor struct {
|
||||
maxPixels int64
|
||||
@@ -83,29 +76,72 @@ func transient(code string, err error) *Failure {
|
||||
return &Failure{Permanent: false, Code: code, Err: err}
|
||||
}
|
||||
|
||||
// Stats separates decode, pixel scaling and the two encoders. A single
|
||||
// processing_ms number cannot answer whether a large image costs more because
|
||||
// of resampling or because of JPEG/WebP compression.
|
||||
type Stats struct {
|
||||
Decode time.Duration
|
||||
Scale time.Duration
|
||||
JPEGEncode time.Duration
|
||||
WebPEncode time.Duration
|
||||
}
|
||||
|
||||
func (s Stats) Total() time.Duration {
|
||||
return s.Decode + s.Scale + s.JPEGEncode + s.WebPEncode
|
||||
}
|
||||
|
||||
// Process decodes the file once and returns all variants in memory. Worker
|
||||
// concurrency and IMAGE_DECODE_MEMORY_MB bound the number of simultaneous
|
||||
// decoded images, which is the memory-heavy part of the experiment.
|
||||
func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, error) {
|
||||
// decoded images, which is the memory-heavy part of the pipeline.
|
||||
func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, Stats, error) {
|
||||
var stats Stats
|
||||
started := time.Now()
|
||||
img, err := p.decode(ctx, file)
|
||||
stats.Decode = time.Since(started)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, stats, err
|
||||
}
|
||||
|
||||
thumbnail, err := renderJPEG("thumbnail", fitOnWhite(img, ThumbnailMax, ThumbnailMax))
|
||||
started = time.Now()
|
||||
thumbnailPixels := fitOnWhite(img, config.ThumbnailMax, config.ThumbnailMax)
|
||||
stats.Scale += time.Since(started)
|
||||
thumbnail, encode, err := renderJPEGTimed("thumbnail", thumbnailPixels)
|
||||
stats.JPEGEncode += encode
|
||||
if err != nil {
|
||||
return nil, permanent("processing_failed", err)
|
||||
}
|
||||
crop, err := renderJPEG("crop", centerSquare(img, CropSize))
|
||||
if err != nil {
|
||||
return nil, permanent("processing_failed", err)
|
||||
}
|
||||
optimized, err := renderWebP(fitOnWhite(img, OptimizedMax, OptimizedMax))
|
||||
if err != nil {
|
||||
return nil, permanent("processing_failed", err)
|
||||
return nil, stats, permanent("processing_failed", err)
|
||||
}
|
||||
|
||||
return []Encoded{thumbnail, crop, optimized}, nil
|
||||
started = time.Now()
|
||||
cropPixels := centerSquare(img, config.CropSize)
|
||||
stats.Scale += time.Since(started)
|
||||
crop, encode, err := renderJPEGTimed("crop", cropPixels)
|
||||
stats.JPEGEncode += encode
|
||||
if err != nil {
|
||||
return nil, stats, permanent("processing_failed", err)
|
||||
}
|
||||
|
||||
started = time.Now()
|
||||
optimizedPixels := fitOnWhite(img, config.OptimizedMax, config.OptimizedMax)
|
||||
stats.Scale += time.Since(started)
|
||||
optimized, encode, err := renderWebPTimed(optimizedPixels)
|
||||
stats.WebPEncode += encode
|
||||
if err != nil {
|
||||
return nil, stats, permanent("processing_failed", err)
|
||||
}
|
||||
|
||||
return []Encoded{thumbnail, crop, optimized}, stats, nil
|
||||
}
|
||||
|
||||
func renderJPEGTimed(name string, img image.Image) (Encoded, time.Duration, error) {
|
||||
started := time.Now()
|
||||
encoded, err := renderJPEG(name, img)
|
||||
return encoded, time.Since(started), err
|
||||
}
|
||||
|
||||
func renderWebPTimed(img image.Image) (Encoded, time.Duration, error) {
|
||||
started := time.Now()
|
||||
encoded, err := renderWebP(img)
|
||||
return encoded, time.Since(started), err
|
||||
}
|
||||
|
||||
func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, error) {
|
||||
@@ -127,7 +163,7 @@ func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, err
|
||||
return nil, permanent("invalid_image", errors.New("image exceeds MAX_IMAGE_PIXELS"))
|
||||
}
|
||||
|
||||
// AGENT: the semaphore is acquired before Decode, not after. A decoder
|
||||
// The semaphore is acquired before Decode, not after. A decoder
|
||||
// allocates roughly width*height*4 bytes; without this gate several 40 MP
|
||||
// files can OOM the worker even with a small HTTP concurrency.
|
||||
weight := pixels * 4
|
||||
@@ -176,7 +212,7 @@ func fitOnWhite(src image.Image, maxWidth, maxHeight int) *image.RGBA {
|
||||
return dst
|
||||
}
|
||||
|
||||
// centerSquare always produces CropSize x CropSize. Unlike thumbnails, the
|
||||
// centerSquare always produces config.CropSize x config.CropSize. Unlike thumbnails, the
|
||||
// contract allows upscaling tiny sources for the crop variant.
|
||||
func centerSquare(src image.Image, size int) *image.RGBA {
|
||||
bounds := src.Bounds()
|
||||
@@ -216,7 +252,7 @@ func fitSize(width, height, maxWidth, maxHeight int) (int, int) {
|
||||
|
||||
func renderJPEG(name string, img image.Image) (Encoded, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil {
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: config.JPEGQuality}); err != nil {
|
||||
return Encoded{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
@@ -232,7 +268,7 @@ func renderJPEG(name string, img image.Image) (Encoded, error) {
|
||||
|
||||
func renderWebP(img image.Image) (Encoded, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := webp.Encode(&buf, img, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil {
|
||||
if err := webp.Encode(&buf, img, webp.Options{Quality: config.WebPQuality, Method: config.WebPMethod}); err != nil {
|
||||
return Encoded{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
|
||||
@@ -34,9 +34,26 @@ func TestFitSizePreservesAspectAndNeverUpscales(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessReportsTiming(t *testing.T) {
|
||||
file := pngFile(t, 320, 200, false)
|
||||
_, stats, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
if err != nil {
|
||||
t.Fatalf("Process: %v", err)
|
||||
}
|
||||
if stats.Decode <= 0 || stats.Scale <= 0 || stats.JPEGEncode <= 0 || stats.WebPEncode <= 0 {
|
||||
t.Fatalf("incomplete timing stats: %+v", stats)
|
||||
}
|
||||
if stats.Total() != stats.Decode+stats.Scale+stats.JPEGEncode+stats.WebPEncode {
|
||||
t.Fatalf("total %v does not match its parts: %+v", stats.Total(), stats)
|
||||
}
|
||||
if stats.Total() <= 0 {
|
||||
t.Fatalf("total timing must be positive: %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessProducesContractVariants(t *testing.T) {
|
||||
file := pngFile(t, 320, 200, false)
|
||||
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
if err != nil {
|
||||
t.Fatalf("Process: %v", err)
|
||||
}
|
||||
@@ -73,7 +90,7 @@ func TestProcessProducesContractVariants(t *testing.T) {
|
||||
|
||||
func TestThumbnailCompositesAlphaOnWhite(t *testing.T) {
|
||||
file := pngFile(t, 64, 64, true)
|
||||
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
if err != nil {
|
||||
t.Fatalf("Process: %v", err)
|
||||
}
|
||||
@@ -92,7 +109,7 @@ func TestEXIFOrientationRotatesBeforeResize(t *testing.T) {
|
||||
if got := jpegOrientation(file); got != 6 {
|
||||
t.Fatalf("jpegOrientation = %d, want 6", got)
|
||||
}
|
||||
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
variants, _, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
if err != nil {
|
||||
t.Fatalf("Process: %v", err)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,18 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Job state machine. Every transition is a compare-and-set on
|
||||
// (job_id, attempt, status), so broker duplicates and stale workers are
|
||||
// harmless.
|
||||
//
|
||||
// queued --ClaimJob--> processing --CompleteJob--> ready
|
||||
// |
|
||||
// +--ScheduleRetry--> queued (future available_at)
|
||||
// +--FailJob--------> failed
|
||||
// +--attempt >= max-> failed/retry_exhausted
|
||||
// processing with an expired lease --RecoverExpiredLeases--> queued/failed
|
||||
// failed --ReplayFailed--> queued (attempt=0, replay_count+1)
|
||||
//
|
||||
// ClaimResult tells the consumer whether this delivery owns the work.
|
||||
type ClaimResult struct {
|
||||
Job model.JobRecord
|
||||
|
||||
@@ -124,7 +124,6 @@ func (w *Worker) Handle(ctx context.Context, message model.Job) error {
|
||||
return w.handleFailure(ctx, claim, err, log)
|
||||
}
|
||||
|
||||
log.Info("attachment ready")
|
||||
if w.cfg.CrashAfterEffect {
|
||||
// Deterministic fault injection for the "written but not acked" window.
|
||||
// The unacked delivery is redelivered after the connection dies.
|
||||
@@ -191,7 +190,7 @@ func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")}
|
||||
}
|
||||
|
||||
encoded, err := w.pipeline.Process(ctx, file)
|
||||
encoded, stats, err := w.pipeline.Process(ctx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -229,6 +228,17 @@ func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog
|
||||
return nil
|
||||
}
|
||||
|
||||
// The timing line is the experiment's source for compression overhead. It
|
||||
// is emitted only after the ready snapshot commits, so a redelivered job
|
||||
// cannot produce two successful samples.
|
||||
log.Info("attachment ready",
|
||||
"decode_us", stats.Decode.Microseconds(),
|
||||
"scale_us", stats.Scale.Microseconds(),
|
||||
"jpeg_encode_us", stats.JPEGEncode.Microseconds(),
|
||||
"webp_encode_us", stats.WebPEncode.Microseconds(),
|
||||
"pipeline_us", stats.Total().Microseconds(),
|
||||
)
|
||||
|
||||
// Staging is removed only after ready commits. A crash before this delete
|
||||
// leaves an object the reconciler will clean after retention.
|
||||
if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
|
||||
Reference in New Issue
Block a user