From 5442dfd3bc0127cc9ca0b8db85e17aacd517175e Mon Sep 17 00:00:00 2001 From: kk10 Date: Mon, 16 Nov 2026 22:06:11 +0300 Subject: [PATCH] feat(worker): add versioned image processing --- cmd/worker/main.go | 3 +- internal/worker/image.go | 30 +++++-- internal/worker/image_process.go | 11 ++- student-tests/image_pipeline_test.go | 126 +++++++++++++++++++++++++++ student.mk | 6 +- 5 files changed, 161 insertions(+), 15 deletions(-) create mode 100644 student-tests/image_pipeline_test.go diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 08bccc29..60c1cd8a 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -31,6 +31,7 @@ var ( func main() { maxPixels := flag.Int("process-image", 0, "process an image from stdin with the given pixel limit") + pipelineVersion := flag.Int("pipeline-version", 1, "pipeline version for process-image") replay := flag.String("replay", "", "replay a failed attachment by UUID; requires retained staging and no existing variants") operator := flag.String("operator", "", "operator requesting replay") reason := flag.String("reason", "", "reason for replay") @@ -38,7 +39,7 @@ func main() { flag.Parse() if *maxPixels > 0 { - if err := worker.ProcessImage(*maxPixels); err != nil { + if err := worker.ProcessImage(*maxPixels, *pipelineVersion); err != nil { if errors.Is(err, worker.ErrInvalidSource) { os.Exit(2) } diff --git a/internal/worker/image.go b/internal/worker/image.go index ae1e7a42..fe6091a4 100644 --- a/internal/worker/image.go +++ b/internal/worker/image.go @@ -21,7 +21,10 @@ import ( "mtusi2026-messenger/internal/db" ) -var ErrInvalidSource = errors.New("invalid source image") +var ( + ErrInvalidSource = errors.New("invalid source image") + ErrUnsupportedPipelineVersion = errors.New("unsupported pipeline version") +) type imageVariant struct { Name db.AttachmentVariantName @@ -78,7 +81,7 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt return nil, err } - images, err := runImageProcess(ctx, tmp, w.maxPixels) + images, err := runImageProcess(ctx, tmp, w.maxPixels, row.PipelineVersion) <-w.decodeSlot if err != nil { return nil, err @@ -120,7 +123,16 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt return variants, nil } -func imageVariants(r io.ReadSeeker, maxPixels int) ([]imageVariant, error) { +func imageVariants(r io.ReadSeeker, maxPixels, pipelineVersion int) ([]imageVariant, error) { + quality := 80 + switch pipelineVersion { + case 1: + case 2: + quality = 75 + default: + return nil, ErrUnsupportedPipelineVersion + } + cfg, format, err := image.DecodeConfig(r) if err != nil || (format != "png" && format != "jpeg") || @@ -152,9 +164,9 @@ func imageVariants(r io.ReadSeeker, maxPixels int) ([]imageVariant, error) { contentType := "image/jpeg" if item.name == db.AttachmentVariantNameOptimized { contentType = "image/webp" - data, err = encodeWebP(item.image) + data, err = encodeWebP(item.image, quality) } else { - data, err = encodeJPEG(item.image) + data, err = encodeJPEG(item.image, 85) } if err != nil { @@ -190,22 +202,22 @@ func fitImage(src image.Image, maxSize int) image.Image { return src } -func encodeJPEG(src image.Image) ([]byte, error) { +func encodeJPEG(src image.Image, quality int) ([]byte, error) { dst := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy())) draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) draw.Draw(dst, dst.Bounds(), src, src.Bounds().Min, draw.Over) var out bytes.Buffer - if err := jpeg.Encode(&out, dst, &jpeg.Options{Quality: 85}); err != nil { + if err := jpeg.Encode(&out, dst, &jpeg.Options{Quality: quality}); err != nil { return nil, err } return out.Bytes(), nil } -func encodeWebP(src image.Image) ([]byte, error) { +func encodeWebP(src image.Image, quality int) ([]byte, error) { var out bytes.Buffer - if err := webp.Encode(&out, src, webp.Options{Quality: 80, Method: 4}); err != nil { + if err := webp.Encode(&out, src, webp.Options{Quality: quality, Method: 4}); err != nil { return nil, err } diff --git a/internal/worker/image_process.go b/internal/worker/image_process.go index 4811d5a8..e0f17b3a 100644 --- a/internal/worker/image_process.go +++ b/internal/worker/image_process.go @@ -10,8 +10,8 @@ import ( "strconv" ) -func ProcessImage(maxPixels int) error { - variants, err := imageVariants(os.Stdin, maxPixels) +func ProcessImage(maxPixels, pipelineVersion int) error { + variants, err := imageVariants(os.Stdin, maxPixels, pipelineVersion) if err != nil { return err } @@ -19,13 +19,16 @@ func ProcessImage(maxPixels int) error { return gob.NewEncoder(os.Stdout).Encode(variants) } -func runImageProcess(ctx context.Context, source *os.File, maxPixels int) ([]imageVariant, error) { +func runImageProcess(ctx context.Context, source *os.File, maxPixels, pipelineVersion int) ([]imageVariant, error) { executable, err := os.Executable() if err != nil { return nil, err } - cmd := exec.CommandContext(ctx, executable, "-process-image", strconv.Itoa(maxPixels)) + cmd := exec.CommandContext(ctx, executable, + "-process-image", strconv.Itoa(maxPixels), + "-pipeline-version", strconv.Itoa(pipelineVersion), + ) cmd.Stdin = source data, err := cmd.Output() if ctx.Err() != nil { diff --git a/student-tests/image_pipeline_test.go b/student-tests/image_pipeline_test.go new file mode 100644 index 00000000..b951f179 --- /dev/null +++ b/student-tests/image_pipeline_test.go @@ -0,0 +1,126 @@ +//go:build pipeline + +package studenttests_test + +import ( + "bytes" + "context" + "encoding/gob" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "golang.org/x/image/webp" +) + +func TestPipelineImageVersions(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + binary := filepath.Join(t.TempDir(), "worker") + build := exec.CommandContext(ctx, "go", "build", "-tags=nodynamic", "-o", binary, "../cmd/worker") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build worker: %v\n%s", err, out) + } + + src := image.NewNRGBA(image.Rect(0, 0, 320, 200)) + for y := 0; y < 200; y++ { + for x := 0; x < 320; x++ { + src.SetNRGBA(x, y, color.NRGBA{ + R: uint8(x*17 + y*3), + G: uint8(x*5 + y*11), + B: uint8(x * y), A: 255, + }) + } + } + + var source bytes.Buffer + if err := png.Encode(&source, src); err != nil { + t.Fatal(err) + } + + sourcePath := filepath.Join(t.TempDir(), "source.png") + if err := os.WriteFile(sourcePath, source.Bytes(), 0600); err != nil { + t.Fatal(err) + } + + // Decode the child process protocol without exporting worker internals. + type variant struct { + Name, ContentType string + Data []byte + Width, Height int + } + process := func(version string) []variant { + t.Helper() + args := []string{"-process-image", "40000000"} + if version != "" { + args = append(args, "-pipeline-version", version) + } + cmd := exec.CommandContext(ctx, binary, args...) + input, err := os.Open(sourcePath) + if err != nil { + t.Fatal(err) + } + defer input.Close() + cmd.Stdin = input + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + t.Fatalf("process v%s: %v: %s", version, err, stderr.String()) + } + + var variants []variant + if err := gob.NewDecoder(bytes.NewReader(out)).Decode(&variants); err != nil { + t.Fatal(err) + } + if len(variants) != 3 { + t.Fatalf("expected three variants, got %d", len(variants)) + } + for i, name := range []string{"thumbnail", "crop", "optimized"} { + v := variants[i] + var cfg image.Config + var err error + width, height, contentType := 256, 160, "image/jpeg" + if name == "optimized" { + width, height, contentType = 320, 200, "image/webp" + cfg, err = webp.DecodeConfig(bytes.NewReader(v.Data)) + } else { + if name == "crop" { + width, height = 128, 128 + } + cfg, err = jpeg.DecodeConfig(bytes.NewReader(v.Data)) + } + if err != nil || v.Name != name || v.ContentType != contentType || + v.Width != width || v.Height != height || cfg.Width != width || cfg.Height != height { + t.Fatalf("invalid v%s %s: metadata=%dx%d, decoded=%dx%d, error=%v", + version, name, v.Width, v.Height, cfg.Width, cfg.Height, err) + } + } + return variants + } + + v1, v2 := process("1"), process("2") + defaultVersion := process("") + for i := range v1 { + if !bytes.Equal(v1[i].Data, defaultVersion[i].Data) { + t.Fatal("default processing differs from v1") + } + wantEqual := v1[i].Name != "optimized" + if bytes.Equal(v1[i].Data, v2[i].Data) != wantEqual { + t.Fatalf("expected only optimized bytes to differ: %s", v1[i].Name) + } + } + + cmd := exec.CommandContext(ctx, binary, "-process-image", "40000000", "-pipeline-version", "3") + cmd.Stdin = bytes.NewReader(source.Bytes()) + if out, err := cmd.Output(); err == nil || len(out) != 0 { + t.Fatal("unsupported version must fail without a result") + } +} diff --git a/student.mk b/student.mk index c5c51fa9..acae1ddd 100644 --- a/student.mk +++ b/student.mk @@ -8,7 +8,7 @@ ACTION ?= stop-one.sh ACTION_AT ?= 10s DRY_RUN ?= true -.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go worker-go +.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go worker-go pipeline-go help-go: @echo "contract-go: contract tests" @@ -17,6 +17,7 @@ help-go: @echo "load-go: load test" @echo "failure-go: dependency failure tests (stops redis/postgres/s3 containers)" @echo "reconcile: attachment audit; DRY_RUN=false to repair" + @echo "pipeline-go: image pipeline versions (no running services required)" contract-go: go test -tags=contract ./student-tests/... -v -args -base-url=$(BASE_URL) @@ -46,3 +47,6 @@ reconcile-test-go: worker-go: go test -count=1 -tags=worker ./student-tests/... -v -args -base-url=$(BASE_URL) + +pipeline-go: + go test -count=1 -tags=nodynamic,pipeline ./student-tests/... -v