feat(worker): add versioned image processing

This commit is contained in:
kk10
2026-11-16 22:06:11 +03:00
parent 6ba7db4b7d
commit 5442dfd3bc
5 changed files with 161 additions and 15 deletions
+2 -1
View File
@@ -31,6 +31,7 @@ var (
func main() {
maxPixels := flag.Int("process-image", 0, "process an image from stdin with the given pixel limit")
pipelineVersion := flag.Int("pipeline-version", 1, "pipeline version for process-image")
replay := flag.String("replay", "", "replay a failed attachment by UUID; requires retained staging and no existing variants")
operator := flag.String("operator", "", "operator requesting replay")
reason := flag.String("reason", "", "reason for replay")
@@ -38,7 +39,7 @@ func main() {
flag.Parse()
if *maxPixels > 0 {
if err := worker.ProcessImage(*maxPixels); err != nil {
if err := worker.ProcessImage(*maxPixels, *pipelineVersion); err != nil {
if errors.Is(err, worker.ErrInvalidSource) {
os.Exit(2)
}
+21 -9
View File
@@ -21,7 +21,10 @@ import (
"mtusi2026-messenger/internal/db"
)
var ErrInvalidSource = errors.New("invalid source image")
var (
ErrInvalidSource = errors.New("invalid source image")
ErrUnsupportedPipelineVersion = errors.New("unsupported pipeline version")
)
type imageVariant struct {
Name db.AttachmentVariantName
@@ -78,7 +81,7 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt
return nil, err
}
images, err := runImageProcess(ctx, tmp, w.maxPixels)
images, err := runImageProcess(ctx, tmp, w.maxPixels, row.PipelineVersion)
<-w.decodeSlot
if err != nil {
return nil, err
@@ -120,7 +123,16 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt
return variants, nil
}
func imageVariants(r io.ReadSeeker, maxPixels int) ([]imageVariant, error) {
func imageVariants(r io.ReadSeeker, maxPixels, pipelineVersion int) ([]imageVariant, error) {
quality := 80
switch pipelineVersion {
case 1:
case 2:
quality = 75
default:
return nil, ErrUnsupportedPipelineVersion
}
cfg, format, err := image.DecodeConfig(r)
if err != nil ||
(format != "png" && format != "jpeg") ||
@@ -152,9 +164,9 @@ func imageVariants(r io.ReadSeeker, maxPixels int) ([]imageVariant, error) {
contentType := "image/jpeg"
if item.name == db.AttachmentVariantNameOptimized {
contentType = "image/webp"
data, err = encodeWebP(item.image)
data, err = encodeWebP(item.image, quality)
} else {
data, err = encodeJPEG(item.image)
data, err = encodeJPEG(item.image, 85)
}
if err != nil {
@@ -190,22 +202,22 @@ func fitImage(src image.Image, maxSize int) image.Image {
return src
}
func encodeJPEG(src image.Image) ([]byte, error) {
func encodeJPEG(src image.Image, quality int) ([]byte, error) {
dst := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
draw.Draw(dst, dst.Bounds(), src, src.Bounds().Min, draw.Over)
var out bytes.Buffer
if err := jpeg.Encode(&out, dst, &jpeg.Options{Quality: 85}); err != nil {
if err := jpeg.Encode(&out, dst, &jpeg.Options{Quality: quality}); err != nil {
return nil, err
}
return out.Bytes(), nil
}
func encodeWebP(src image.Image) ([]byte, error) {
func encodeWebP(src image.Image, quality int) ([]byte, error) {
var out bytes.Buffer
if err := webp.Encode(&out, src, webp.Options{Quality: 80, Method: 4}); err != nil {
if err := webp.Encode(&out, src, webp.Options{Quality: quality, Method: 4}); err != nil {
return nil, err
}
+7 -4
View File
@@ -10,8 +10,8 @@ import (
"strconv"
)
func ProcessImage(maxPixels int) error {
variants, err := imageVariants(os.Stdin, maxPixels)
func ProcessImage(maxPixels, pipelineVersion int) error {
variants, err := imageVariants(os.Stdin, maxPixels, pipelineVersion)
if err != nil {
return err
}
@@ -19,13 +19,16 @@ func ProcessImage(maxPixels int) error {
return gob.NewEncoder(os.Stdout).Encode(variants)
}
func runImageProcess(ctx context.Context, source *os.File, maxPixels int) ([]imageVariant, error) {
func runImageProcess(ctx context.Context, source *os.File, maxPixels, pipelineVersion int) ([]imageVariant, error) {
executable, err := os.Executable()
if err != nil {
return nil, err
}
cmd := exec.CommandContext(ctx, executable, "-process-image", strconv.Itoa(maxPixels))
cmd := exec.CommandContext(ctx, executable,
"-process-image", strconv.Itoa(maxPixels),
"-pipeline-version", strconv.Itoa(pipelineVersion),
)
cmd.Stdin = source
data, err := cmd.Output()
if ctx.Err() != nil {
+126
View File
@@ -0,0 +1,126 @@
//go:build pipeline
package studenttests_test
import (
"bytes"
"context"
"encoding/gob"
"image"
"image/color"
"image/jpeg"
"image/png"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
"golang.org/x/image/webp"
)
func TestPipelineImageVersions(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
binary := filepath.Join(t.TempDir(), "worker")
build := exec.CommandContext(ctx, "go", "build", "-tags=nodynamic", "-o", binary, "../cmd/worker")
if out, err := build.CombinedOutput(); err != nil {
t.Fatalf("build worker: %v\n%s", err, out)
}
src := image.NewNRGBA(image.Rect(0, 0, 320, 200))
for y := 0; y < 200; y++ {
for x := 0; x < 320; x++ {
src.SetNRGBA(x, y, color.NRGBA{
R: uint8(x*17 + y*3),
G: uint8(x*5 + y*11),
B: uint8(x * y), A: 255,
})
}
}
var source bytes.Buffer
if err := png.Encode(&source, src); err != nil {
t.Fatal(err)
}
sourcePath := filepath.Join(t.TempDir(), "source.png")
if err := os.WriteFile(sourcePath, source.Bytes(), 0600); err != nil {
t.Fatal(err)
}
// Decode the child process protocol without exporting worker internals.
type variant struct {
Name, ContentType string
Data []byte
Width, Height int
}
process := func(version string) []variant {
t.Helper()
args := []string{"-process-image", "40000000"}
if version != "" {
args = append(args, "-pipeline-version", version)
}
cmd := exec.CommandContext(ctx, binary, args...)
input, err := os.Open(sourcePath)
if err != nil {
t.Fatal(err)
}
defer input.Close()
cmd.Stdin = input
var stderr bytes.Buffer
cmd.Stderr = &stderr
out, err := cmd.Output()
if err != nil {
t.Fatalf("process v%s: %v: %s", version, err, stderr.String())
}
var variants []variant
if err := gob.NewDecoder(bytes.NewReader(out)).Decode(&variants); err != nil {
t.Fatal(err)
}
if len(variants) != 3 {
t.Fatalf("expected three variants, got %d", len(variants))
}
for i, name := range []string{"thumbnail", "crop", "optimized"} {
v := variants[i]
var cfg image.Config
var err error
width, height, contentType := 256, 160, "image/jpeg"
if name == "optimized" {
width, height, contentType = 320, 200, "image/webp"
cfg, err = webp.DecodeConfig(bytes.NewReader(v.Data))
} else {
if name == "crop" {
width, height = 128, 128
}
cfg, err = jpeg.DecodeConfig(bytes.NewReader(v.Data))
}
if err != nil || v.Name != name || v.ContentType != contentType ||
v.Width != width || v.Height != height || cfg.Width != width || cfg.Height != height {
t.Fatalf("invalid v%s %s: metadata=%dx%d, decoded=%dx%d, error=%v",
version, name, v.Width, v.Height, cfg.Width, cfg.Height, err)
}
}
return variants
}
v1, v2 := process("1"), process("2")
defaultVersion := process("")
for i := range v1 {
if !bytes.Equal(v1[i].Data, defaultVersion[i].Data) {
t.Fatal("default processing differs from v1")
}
wantEqual := v1[i].Name != "optimized"
if bytes.Equal(v1[i].Data, v2[i].Data) != wantEqual {
t.Fatalf("expected only optimized bytes to differ: %s", v1[i].Name)
}
}
cmd := exec.CommandContext(ctx, binary, "-process-image", "40000000", "-pipeline-version", "3")
cmd.Stdin = bytes.NewReader(source.Bytes())
if out, err := cmd.Output(); err == nil || len(out) != 0 {
t.Fatal("unsupported version must fail without a result")
}
}
+5 -1
View File
@@ -8,7 +8,7 @@ ACTION ?= stop-one.sh
ACTION_AT ?= 10s
DRY_RUN ?= true
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go worker-go
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go worker-go pipeline-go
help-go:
@echo "contract-go: contract tests"
@@ -17,6 +17,7 @@ help-go:
@echo "load-go: load test"
@echo "failure-go: dependency failure tests (stops redis/postgres/s3 containers)"
@echo "reconcile: attachment audit; DRY_RUN=false to repair"
@echo "pipeline-go: image pipeline versions (no running services required)"
contract-go:
go test -tags=contract ./student-tests/... -v -args -base-url=$(BASE_URL)
@@ -46,3 +47,6 @@ reconcile-test-go:
worker-go:
go test -count=1 -tags=worker ./student-tests/... -v -args -base-url=$(BASE_URL)
pipeline-go:
go test -count=1 -tags=nodynamic,pipeline ./student-tests/... -v