feat(worker): add optimized WebP variant

This commit is contained in:
kk10
2026-11-16 13:29:09 +03:00
parent 73d57e31a5
commit be10119910
6 changed files with 115 additions and 39 deletions
+1 -1
View File
@@ -5,7 +5,7 @@ COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o /out/worker ./cmd/worker
RUN CGO_ENABLED=0 GOOS=linux go build -tags=nodynamic -o /out/worker ./cmd/worker
FROM gcr.io/distroless/static-debian12:nonroot
WORKDIR /app
+3 -1
View File
@@ -9,6 +9,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
github.com/aws/smithy-go v1.28.7
github.com/disintegration/imaging v1.6.2
github.com/gen2brain/webp v0.6.4
github.com/go-chi/chi/v5 v5.5.0
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
@@ -16,6 +17,7 @@ require (
github.com/rabbitmq/amqp091-go v1.10.0
github.com/redis/go-redis/v9 v9.23.0
golang.org/x/crypto v0.58.0
golang.org/x/image v0.48.0
)
require (
@@ -33,11 +35,11 @@ require (
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/ebitengine/purego v0.10.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
go.uber.org/atomic v1.12.0 // indirect
golang.org/x/image v0.48.0 // indirect
golang.org/x/sync v0.25.0 // indirect
golang.org/x/sys v0.50.0 // indirect
golang.org/x/text v0.44.0 // indirect
+4
View File
@@ -43,6 +43,10 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU=
github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE=
github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+37 -18
View File
@@ -16,6 +16,7 @@ import (
"strconv"
"github.com/disintegration/imaging"
"github.com/gen2brain/webp"
"mtusi2026-messenger/internal/db"
)
@@ -25,6 +26,7 @@ var errInvalidSource = errors.New("invalid source image")
type imageVariant struct {
name db.AttachmentVariantName
data []byte
contentType string
width, height int
}
@@ -97,7 +99,7 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt
variants := make([]db.CreateAttachmentVariantParams, 0, len(images))
for _, img := range images {
key := "variants/" + row.ID.String() + "/v" + strconv.Itoa(row.PipelineVersion) + "/" + string(img.name)
if err := w.s3.Put(ctx, key, bytes.NewReader(img.data), int64(len(img.data)), "image/jpeg"); err != nil {
if err := w.s3.Put(ctx, key, bytes.NewReader(img.data), int64(len(img.data)), img.contentType); err != nil {
return nil, err
}
@@ -107,7 +109,7 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAtt
PipelineVersion: row.PipelineVersion,
Variant: img.name,
S3Key: key,
ContentType: "image/jpeg",
ContentType: img.contentType,
SizeBytes: len(img.data),
Width: img.width,
Height: img.height,
@@ -137,41 +139,49 @@ func imageVariants(r io.ReadSeeker, maxPixels int) ([]imageVariant, error) {
return nil, errInvalidSource
}
thumb := thumbnail(src)
crop := imaging.Fill(src, 128, 128, imaging.Center, imaging.Lanczos)
variants := make([]imageVariant, 0, 2)
variants := make([]imageVariant, 0, 3)
for _, item := range []struct {
name db.AttachmentVariantName
image image.Image
}{
{db.AttachmentVariantNameThumbnail, thumb},
{db.AttachmentVariantNameCrop, crop},
{db.AttachmentVariantNameThumbnail, fitImage(src, 256)},
{db.AttachmentVariantNameCrop, imaging.Fill(src, 128, 128, imaging.Center, imaging.Lanczos)},
{db.AttachmentVariantNameOptimized, fitImage(src, 1280)},
} {
data, err := encodeJPEG(item.image)
var data []byte
contentType := "image/jpeg"
if item.name == db.AttachmentVariantNameOptimized {
contentType = "image/webp"
data, err = encodeWebP(item.image)
} else {
data, err = encodeJPEG(item.image)
}
if err != nil {
return nil, err
}
variants = append(variants, imageVariant{
name: item.name,
data: data,
width: item.image.Bounds().Dx(),
height: item.image.Bounds().Dy(),
name: item.name,
contentType: contentType,
data: data,
width: item.image.Bounds().Dx(),
height: item.image.Bounds().Dy(),
})
}
return variants, nil
}
func thumbnail(src image.Image) image.Image {
func fitImage(src image.Image, maxSize int) image.Image {
width, height := src.Bounds().Dx(), src.Bounds().Dy()
if width > 256 || height > 256 {
if width > maxSize || height > maxSize {
if width >= height {
height = max(1, height*256/width)
width = 256
height = max(1, height*maxSize/width)
width = maxSize
} else {
width = max(1, width*256/height)
height = 256
width = max(1, width*maxSize/height)
height = maxSize
}
return imaging.Resize(src, width, height, imaging.Lanczos)
@@ -192,3 +202,12 @@ func encodeJPEG(src image.Image) ([]byte, error) {
return out.Bytes(), nil
}
func encodeWebP(src image.Image) ([]byte, error) {
var out bytes.Buffer
if err := webp.Encode(&out, src, webp.Options{Quality: 80, Method: 4}); err != nil {
return nil, err
}
return out.Bytes(), nil
}
@@ -4,7 +4,9 @@ package studenttests_test
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"image"
"image/color"
"image/jpeg"
@@ -14,6 +16,8 @@ import (
"testing"
"time"
"golang.org/x/image/webp"
"mtusi2026-messenger/internal/dto"
)
@@ -54,7 +58,7 @@ func TestThumbnail(t *testing.T) {
}
}
func TestThumbnailEXIF(t *testing.T) {
func TestVariantsEXIF(t *testing.T) {
src := image.NewRGBA(image.Rect(0, 0, 40, 20))
for y := 0; y < 20; y++ {
@@ -86,24 +90,27 @@ func TestThumbnailEXIF(t *testing.T) {
data = append(data, segment...)
data = append(data, exif...)
data = append(data, original.Bytes()[2:]...)
result, width, height := processedVariant(t, data, "image/jpeg", "thumbnail")
for _, variant := range []string{"thumbnail", "optimized"} {
t.Run(variant, func(t *testing.T) {
result, width, height := processedVariant(t, data, "image/jpeg", variant)
if width != 20 || height != 40 {
t.Fatalf("orientation ignored: %dx%d", width, height)
}
if width != 20 || height != 40 {
t.Fatalf("orientation ignored: %dx%d", width, height)
}
decoded, _, err := image.Decode(bytes.NewReader(result))
if err != nil {
t.Fatal(err)
}
decoded, err := jpeg.Decode(bytes.NewReader(result))
if err != nil {
t.Fatal(err)
}
r, _, b, _ := decoded.At(10, 5).RGBA()
if r <= b {
t.Fatal("incorrect EXIF rotation")
}
r, _, b, _ := decoded.At(10, 5).RGBA()
if r <= b {
t.Fatal("incorrect EXIF rotation")
}
if bytes.Contains(result, []byte("Exif")) {
t.Fatal("EXIF retained")
if bytes.Contains(result, []byte("Exif")) || bytes.Contains(result, []byte("EXIF")) {
t.Fatal("EXIF retained")
}
})
}
}
@@ -128,6 +135,12 @@ func processedVariant(t *testing.T, source []byte, contentType, variant string)
t.Fatal(err)
}
sum := sha256.Sum256(data)
if len(data) != v.SizeBytes || hex.EncodeToString(sum[:]) != v.Sha256 ||
resp.Header.Get("Content-Type") != string(v.ContentType) {
t.Fatal("variant content disagrees with metadata")
}
return data, v.Width, v.Height
}
t.Fatalf("ready attachment has no %s", variant)
@@ -180,3 +193,34 @@ func TestCrop(t *testing.T) {
})
}
}
func TestOptimized(t *testing.T) {
for _, tc := range []struct {
name string
width, height, wantWidth, wantHeight int
}{
{"landscape", 1600, 1000, 1280, 800},
{"portrait", 1000, 1600, 800, 1280},
{"small", 12, 7, 12, 7},
} {
t.Run(tc.name, func(t *testing.T) {
var source bytes.Buffer
if err := png.Encode(&source, image.NewNRGBA(image.Rect(0, 0, tc.width, tc.height))); err != nil {
t.Fatal(err)
}
data, width, height := processedVariant(t, source.Bytes(), "image/png", "optimized")
decoded, err := webp.Decode(bytes.NewReader(data))
if err != nil {
t.Fatal(err)
}
if width != tc.wantWidth || height != tc.wantHeight ||
decoded.Bounds().Dx() != width || decoded.Bounds().Dy() != height {
t.Fatalf("unexpected optimized dimensions: metadata=%dx%d, WebP=%v", width, height, decoded.Bounds())
}
_, _, _, alpha := decoded.At(0, 0).RGBA()
if alpha != 0 {
t.Fatal("optimized image lost transparency")
}
})
}
}
+10 -3
View File
@@ -17,6 +17,7 @@ import (
"time"
"github.com/google/uuid"
"golang.org/x/image/webp"
"mtusi2026-messenger/internal/dto"
)
@@ -160,17 +161,23 @@ func TestWorkerPersistenceAndDuplicate(t *testing.T) {
if err != nil || cfg.Width != 128 || cfg.Height != 128 {
t.Fatal("invalid JPEG crop")
}
// Re-publish the same intent, modeling duplicate delivery after a lost publisher confirmation.
optimized := workerContent(t, user, att.ID, "optimized")
cfg, err = webp.DecodeConfig(bytes.NewReader(optimized))
if err != nil || cfg.Width != 320 || cfg.Height != 200 {
t.Fatal("invalid optimized WebP")
}
workerSQL(t, "UPDATE outbox SET published_at=NULL WHERE "+where)
workerWait(t, "duplicate publication", 15*time.Second, func() bool {
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
})
workerWait(t, "duplicate acknowledgement", 15*time.Second, func() bool { return workerQueue(t) == "0:0" })
duplicate := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
if !reflect.DeepEqual(ready, duplicate) || workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "2" {
if !reflect.DeepEqual(ready, duplicate) || workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "3" {
t.Fatal("duplicate job changed metadata or duplicated variants")
}
if !bytes.Equal(workerContent(t, user, att.ID, "crop"), crop) ||
if !bytes.Equal(workerContent(t, user, att.ID, "optimized"), optimized) ||
!bytes.Equal(workerContent(t, user, att.ID, "crop"), crop) ||
!bytes.Equal(workerContent(t, user, att.ID, "thumbnail"), thumb) ||
!bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
t.Fatal("duplicate changed content")