forked from mtusi2026/mtusi2026lab5
feat(worker): add versioned image processing
This commit is contained in:
+2
-1
@@ -31,6 +31,7 @@ var (
|
||||
|
||||
func main() {
|
||||
maxPixels := flag.Int("process-image", 0, "process an image from stdin with the given pixel limit")
|
||||
pipelineVersion := flag.Int("pipeline-version", 1, "pipeline version for process-image")
|
||||
replay := flag.String("replay", "", "replay a failed attachment by UUID; requires retained staging and no existing variants")
|
||||
operator := flag.String("operator", "", "operator requesting replay")
|
||||
reason := flag.String("reason", "", "reason for replay")
|
||||
@@ -38,7 +39,7 @@ func main() {
|
||||
flag.Parse()
|
||||
|
||||
if *maxPixels > 0 {
|
||||
if err := worker.ProcessImage(*maxPixels); err != nil {
|
||||
if err := worker.ProcessImage(*maxPixels, *pipelineVersion); err != nil {
|
||||
if errors.Is(err, worker.ErrInvalidSource) {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
@@ -21,7 +21,10 @@ import (
|
||||
"mtusi2026-messenger/internal/db"
|
||||
)
|
||||
|
||||
var ErrInvalidSource = errors.New("invalid source image")
|
||||
var (
|
||||
ErrInvalidSource = errors.New("invalid source image")
|
||||
ErrUnsupportedPipelineVersion = errors.New("unsupported pipeline version")
|
||||
)
|
||||
|
||||
type imageVariant struct {
|
||||
Name db.AttachmentVariantName
|
||||
@@ -78,7 +81,7 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt
|
||||
return nil, err
|
||||
}
|
||||
|
||||
images, err := runImageProcess(ctx, tmp, w.maxPixels)
|
||||
images, err := runImageProcess(ctx, tmp, w.maxPixels, row.PipelineVersion)
|
||||
<-w.decodeSlot
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -120,7 +123,16 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt
|
||||
return variants, nil
|
||||
}
|
||||
|
||||
func imageVariants(r io.ReadSeeker, maxPixels int) ([]imageVariant, error) {
|
||||
func imageVariants(r io.ReadSeeker, maxPixels, pipelineVersion int) ([]imageVariant, error) {
|
||||
quality := 80
|
||||
switch pipelineVersion {
|
||||
case 1:
|
||||
case 2:
|
||||
quality = 75
|
||||
default:
|
||||
return nil, ErrUnsupportedPipelineVersion
|
||||
}
|
||||
|
||||
cfg, format, err := image.DecodeConfig(r)
|
||||
if err != nil ||
|
||||
(format != "png" && format != "jpeg") ||
|
||||
@@ -152,9 +164,9 @@ func imageVariants(r io.ReadSeeker, maxPixels int) ([]imageVariant, error) {
|
||||
contentType := "image/jpeg"
|
||||
if item.name == db.AttachmentVariantNameOptimized {
|
||||
contentType = "image/webp"
|
||||
data, err = encodeWebP(item.image)
|
||||
data, err = encodeWebP(item.image, quality)
|
||||
} else {
|
||||
data, err = encodeJPEG(item.image)
|
||||
data, err = encodeJPEG(item.image, 85)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -190,22 +202,22 @@ func fitImage(src image.Image, maxSize int) image.Image {
|
||||
return src
|
||||
}
|
||||
|
||||
func encodeJPEG(src image.Image) ([]byte, error) {
|
||||
func encodeJPEG(src image.Image, quality int) ([]byte, error) {
|
||||
dst := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
|
||||
draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
|
||||
draw.Draw(dst, dst.Bounds(), src, src.Bounds().Min, draw.Over)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := jpeg.Encode(&out, dst, &jpeg.Options{Quality: 85}); err != nil {
|
||||
if err := jpeg.Encode(&out, dst, &jpeg.Options{Quality: quality}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
|
||||
func encodeWebP(src image.Image) ([]byte, error) {
|
||||
func encodeWebP(src image.Image, quality int) ([]byte, error) {
|
||||
var out bytes.Buffer
|
||||
if err := webp.Encode(&out, src, webp.Options{Quality: 80, Method: 4}); err != nil {
|
||||
if err := webp.Encode(&out, src, webp.Options{Quality: quality, Method: 4}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func ProcessImage(maxPixels int) error {
|
||||
variants, err := imageVariants(os.Stdin, maxPixels)
|
||||
func ProcessImage(maxPixels, pipelineVersion int) error {
|
||||
variants, err := imageVariants(os.Stdin, maxPixels, pipelineVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -19,13 +19,16 @@ func ProcessImage(maxPixels int) error {
|
||||
return gob.NewEncoder(os.Stdout).Encode(variants)
|
||||
}
|
||||
|
||||
func runImageProcess(ctx context.Context, source *os.File, maxPixels int) ([]imageVariant, error) {
|
||||
func runImageProcess(ctx context.Context, source *os.File, maxPixels, pipelineVersion int) ([]imageVariant, error) {
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, executable, "-process-image", strconv.Itoa(maxPixels))
|
||||
cmd := exec.CommandContext(ctx, executable,
|
||||
"-process-image", strconv.Itoa(maxPixels),
|
||||
"-pipeline-version", strconv.Itoa(pipelineVersion),
|
||||
)
|
||||
cmd.Stdin = source
|
||||
data, err := cmd.Output()
|
||||
if ctx.Err() != nil {
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
//go:build pipeline
|
||||
|
||||
package studenttests_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/image/webp"
|
||||
)
|
||||
|
||||
func TestPipelineImageVersions(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
binary := filepath.Join(t.TempDir(), "worker")
|
||||
build := exec.CommandContext(ctx, "go", "build", "-tags=nodynamic", "-o", binary, "../cmd/worker")
|
||||
if out, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build worker: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
src := image.NewNRGBA(image.Rect(0, 0, 320, 200))
|
||||
for y := 0; y < 200; y++ {
|
||||
for x := 0; x < 320; x++ {
|
||||
src.SetNRGBA(x, y, color.NRGBA{
|
||||
R: uint8(x*17 + y*3),
|
||||
G: uint8(x*5 + y*11),
|
||||
B: uint8(x * y), A: 255,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var source bytes.Buffer
|
||||
if err := png.Encode(&source, src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sourcePath := filepath.Join(t.TempDir(), "source.png")
|
||||
if err := os.WriteFile(sourcePath, source.Bytes(), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Decode the child process protocol without exporting worker internals.
|
||||
type variant struct {
|
||||
Name, ContentType string
|
||||
Data []byte
|
||||
Width, Height int
|
||||
}
|
||||
process := func(version string) []variant {
|
||||
t.Helper()
|
||||
args := []string{"-process-image", "40000000"}
|
||||
if version != "" {
|
||||
args = append(args, "-pipeline-version", version)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
input, err := os.Open(sourcePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer input.Close()
|
||||
cmd.Stdin = input
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatalf("process v%s: %v: %s", version, err, stderr.String())
|
||||
}
|
||||
|
||||
var variants []variant
|
||||
if err := gob.NewDecoder(bytes.NewReader(out)).Decode(&variants); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(variants) != 3 {
|
||||
t.Fatalf("expected three variants, got %d", len(variants))
|
||||
}
|
||||
for i, name := range []string{"thumbnail", "crop", "optimized"} {
|
||||
v := variants[i]
|
||||
var cfg image.Config
|
||||
var err error
|
||||
width, height, contentType := 256, 160, "image/jpeg"
|
||||
if name == "optimized" {
|
||||
width, height, contentType = 320, 200, "image/webp"
|
||||
cfg, err = webp.DecodeConfig(bytes.NewReader(v.Data))
|
||||
} else {
|
||||
if name == "crop" {
|
||||
width, height = 128, 128
|
||||
}
|
||||
cfg, err = jpeg.DecodeConfig(bytes.NewReader(v.Data))
|
||||
}
|
||||
if err != nil || v.Name != name || v.ContentType != contentType ||
|
||||
v.Width != width || v.Height != height || cfg.Width != width || cfg.Height != height {
|
||||
t.Fatalf("invalid v%s %s: metadata=%dx%d, decoded=%dx%d, error=%v",
|
||||
version, name, v.Width, v.Height, cfg.Width, cfg.Height, err)
|
||||
}
|
||||
}
|
||||
return variants
|
||||
}
|
||||
|
||||
v1, v2 := process("1"), process("2")
|
||||
defaultVersion := process("")
|
||||
for i := range v1 {
|
||||
if !bytes.Equal(v1[i].Data, defaultVersion[i].Data) {
|
||||
t.Fatal("default processing differs from v1")
|
||||
}
|
||||
wantEqual := v1[i].Name != "optimized"
|
||||
if bytes.Equal(v1[i].Data, v2[i].Data) != wantEqual {
|
||||
t.Fatalf("expected only optimized bytes to differ: %s", v1[i].Name)
|
||||
}
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, binary, "-process-image", "40000000", "-pipeline-version", "3")
|
||||
cmd.Stdin = bytes.NewReader(source.Bytes())
|
||||
if out, err := cmd.Output(); err == nil || len(out) != 0 {
|
||||
t.Fatal("unsupported version must fail without a result")
|
||||
}
|
||||
}
|
||||
+5
-1
@@ -8,7 +8,7 @@ ACTION ?= stop-one.sh
|
||||
ACTION_AT ?= 10s
|
||||
DRY_RUN ?= true
|
||||
|
||||
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go worker-go
|
||||
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go worker-go pipeline-go
|
||||
|
||||
help-go:
|
||||
@echo "contract-go: contract tests"
|
||||
@@ -17,6 +17,7 @@ help-go:
|
||||
@echo "load-go: load test"
|
||||
@echo "failure-go: dependency failure tests (stops redis/postgres/s3 containers)"
|
||||
@echo "reconcile: attachment audit; DRY_RUN=false to repair"
|
||||
@echo "pipeline-go: image pipeline versions (no running services required)"
|
||||
|
||||
contract-go:
|
||||
go test -tags=contract ./student-tests/... -v -args -base-url=$(BASE_URL)
|
||||
@@ -46,3 +47,6 @@ reconcile-test-go:
|
||||
|
||||
worker-go:
|
||||
go test -count=1 -tags=worker ./student-tests/... -v -args -base-url=$(BASE_URL)
|
||||
|
||||
pipeline-go:
|
||||
go test -count=1 -tags=nodynamic,pipeline ./student-tests/... -v
|
||||
|
||||
Reference in New Issue
Block a user