forked from mtusi2026/mtusi2026lab5
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
253 lines
7.2 KiB
Go
253 lines
7.2 KiB
Go
// Package pipeline turns one validated original image into the required lab5
|
|
// variants. It is deterministic and side-effect free so a broker redelivery can
|
|
// simply overwrite the same keys with the same bytes.
|
|
package pipeline
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
"image/color"
|
|
"image/jpeg"
|
|
_ "image/png"
|
|
"os"
|
|
|
|
"github.com/gen2brain/webp"
|
|
"golang.org/x/image/draw"
|
|
"golang.org/x/sync/semaphore"
|
|
)
|
|
|
|
const (
|
|
ThumbnailMax = 256
|
|
CropSize = 128
|
|
OptimizedMax = 1280
|
|
|
|
// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is
|
|
// copied byte-for-byte and keeps its metadata and checksum.
|
|
JPEGQuality = 85
|
|
WebPQuality = 82
|
|
WebPMethod = 4
|
|
)
|
|
|
|
type Processor struct {
|
|
maxPixels int64
|
|
budgetBytes int64
|
|
decodeBudget *semaphore.Weighted
|
|
}
|
|
|
|
// Encoded is one variant ready to be uploaded.
|
|
type Encoded struct {
|
|
Name string
|
|
ContentType string
|
|
Bytes []byte
|
|
Width int
|
|
Height int
|
|
SHA256 string
|
|
}
|
|
|
|
func New(maxPixels, decodeBudgetBytes int64) *Processor {
|
|
return &Processor{
|
|
maxPixels: maxPixels,
|
|
budgetBytes: decodeBudgetBytes,
|
|
decodeBudget: semaphore.NewWeighted(decodeBudgetBytes),
|
|
}
|
|
}
|
|
|
|
// Failure classifies pipeline errors for the retry policy. A corrupt image is
|
|
// permanent (retrying cannot help); I/O, lease expiry and dependency errors are
|
|
// transient.
|
|
type Failure struct {
|
|
Permanent bool
|
|
Code string
|
|
Err error
|
|
}
|
|
|
|
func (f *Failure) Error() string {
|
|
if f.Err == nil {
|
|
return f.Code
|
|
}
|
|
return f.Code + ": " + f.Err.Error()
|
|
}
|
|
|
|
func (f *Failure) Unwrap() error { return f.Err }
|
|
|
|
func permanent(code string, err error) *Failure {
|
|
return &Failure{Permanent: true, Code: code, Err: err}
|
|
}
|
|
|
|
func transient(code string, err error) *Failure {
|
|
return &Failure{Permanent: false, Code: code, Err: err}
|
|
}
|
|
|
|
// Process decodes the file once and returns all variants in memory. Worker
|
|
// concurrency and IMAGE_DECODE_MEMORY_MB bound the number of simultaneous
|
|
// decoded images, which is the memory-heavy part of the experiment.
|
|
func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, error) {
|
|
img, err := p.decode(ctx, file)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
thumbnail, err := renderJPEG("thumbnail", fitOnWhite(img, ThumbnailMax, ThumbnailMax))
|
|
if err != nil {
|
|
return nil, permanent("processing_failed", err)
|
|
}
|
|
crop, err := renderJPEG("crop", centerSquare(img, CropSize))
|
|
if err != nil {
|
|
return nil, permanent("processing_failed", err)
|
|
}
|
|
optimized, err := renderWebP(fitOnWhite(img, OptimizedMax, OptimizedMax))
|
|
if err != nil {
|
|
return nil, permanent("processing_failed", err)
|
|
}
|
|
|
|
return []Encoded{thumbnail, crop, optimized}, nil
|
|
}
|
|
|
|
func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, error) {
|
|
if _, err := file.Seek(0, 0); err != nil {
|
|
return nil, transient("storage_read_failed", err)
|
|
}
|
|
cfg, format, err := image.DecodeConfig(file)
|
|
if err != nil {
|
|
return nil, permanent("invalid_image", err)
|
|
}
|
|
if format != "png" && format != "jpeg" {
|
|
return nil, permanent("invalid_image", fmt.Errorf("unsupported format %q", format))
|
|
}
|
|
if cfg.Width <= 0 || cfg.Height <= 0 {
|
|
return nil, permanent("invalid_image", errors.New("empty image bounds"))
|
|
}
|
|
pixels := int64(cfg.Width) * int64(cfg.Height)
|
|
if pixels > p.maxPixels {
|
|
return nil, permanent("invalid_image", errors.New("image exceeds MAX_IMAGE_PIXELS"))
|
|
}
|
|
|
|
// AGENT: the semaphore is acquired before Decode, not after. A decoder
|
|
// allocates roughly width*height*4 bytes; without this gate several 40 MP
|
|
// files can OOM the worker even with a small HTTP concurrency.
|
|
weight := pixels * 4
|
|
if weight > p.budgetBytes {
|
|
return nil, permanent("invalid_image", errors.New("image exceeds decode memory budget"))
|
|
}
|
|
if err = p.decodeBudget.Acquire(ctx, weight); err != nil {
|
|
return nil, transient("processing_timeout", err)
|
|
}
|
|
defer p.decodeBudget.Release(weight)
|
|
|
|
orientation := jpegOrientation(file)
|
|
if _, err = file.Seek(0, 0); err != nil {
|
|
return nil, transient("storage_read_failed", err)
|
|
}
|
|
img, decodeErr := safeDecode(file)
|
|
if decodeErr != nil {
|
|
return nil, permanent("invalid_image", decodeErr)
|
|
}
|
|
bounds := img.Bounds()
|
|
if bounds.Dx() != cfg.Width || bounds.Dy() != cfg.Height {
|
|
return nil, permanent("invalid_image", errors.New("decoded bounds differ from header"))
|
|
}
|
|
return orient(img, orientation), nil
|
|
}
|
|
|
|
func safeDecode(file *os.File) (img image.Image, err error) {
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
img = nil
|
|
err = fmt.Errorf("image decoder panic: %v", recovered)
|
|
}
|
|
}()
|
|
img, _, err = image.Decode(file)
|
|
return img, err
|
|
}
|
|
|
|
// fitOnWhite scales down to fit maxWidth x maxHeight, never up. The source is
|
|
// composited over white, which is the contract for JPEG variants with alpha.
|
|
func fitOnWhite(src image.Image, maxWidth, maxHeight int) *image.RGBA {
|
|
bounds := src.Bounds()
|
|
width, height := fitSize(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight)
|
|
dst := image.NewRGBA(image.Rect(0, 0, width, height))
|
|
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
|
|
draw.CatmullRom.Scale(dst, dst.Bounds(), src, bounds, draw.Over, nil)
|
|
return dst
|
|
}
|
|
|
|
// centerSquare always produces CropSize x CropSize. Unlike thumbnails, the
|
|
// contract allows upscaling tiny sources for the crop variant.
|
|
func centerSquare(src image.Image, size int) *image.RGBA {
|
|
bounds := src.Bounds()
|
|
side := bounds.Dx()
|
|
if bounds.Dy() < side {
|
|
side = bounds.Dy()
|
|
}
|
|
x := bounds.Min.X + (bounds.Dx()-side)/2
|
|
y := bounds.Min.Y + (bounds.Dy()-side)/2
|
|
cropRect := image.Rect(x, y, x+side, y+side)
|
|
|
|
dst := image.NewRGBA(image.Rect(0, 0, size, size))
|
|
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
|
|
draw.CatmullRom.Scale(dst, dst.Bounds(), src, cropRect, draw.Over, nil)
|
|
return dst
|
|
}
|
|
|
|
// fitSize preserves aspect ratio, rounds down and clamps to at least 1 px.
|
|
// It deliberately never enlarges a small source.
|
|
func fitSize(width, height, maxWidth, maxHeight int) (int, int) {
|
|
if width <= maxWidth && height <= maxHeight {
|
|
return width, height
|
|
}
|
|
if width*maxHeight >= height*maxWidth {
|
|
newHeight := height * maxWidth / width
|
|
if newHeight < 1 {
|
|
newHeight = 1
|
|
}
|
|
return maxWidth, newHeight
|
|
}
|
|
newWidth := width * maxHeight / height
|
|
if newWidth < 1 {
|
|
newWidth = 1
|
|
}
|
|
return newWidth, maxHeight
|
|
}
|
|
|
|
func renderJPEG(name string, img image.Image) (Encoded, error) {
|
|
var buf bytes.Buffer
|
|
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil {
|
|
return Encoded{}, err
|
|
}
|
|
bounds := img.Bounds()
|
|
return Encoded{
|
|
Name: name,
|
|
ContentType: "image/jpeg",
|
|
Bytes: buf.Bytes(),
|
|
Width: bounds.Dx(),
|
|
Height: bounds.Dy(),
|
|
SHA256: digest(buf.Bytes()),
|
|
}, nil
|
|
}
|
|
|
|
func renderWebP(img image.Image) (Encoded, error) {
|
|
var buf bytes.Buffer
|
|
if err := webp.Encode(&buf, img, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil {
|
|
return Encoded{}, err
|
|
}
|
|
bounds := img.Bounds()
|
|
return Encoded{
|
|
Name: "optimized",
|
|
ContentType: "image/webp",
|
|
Bytes: buf.Bytes(),
|
|
Width: bounds.Dx(),
|
|
Height: bounds.Dy(),
|
|
SHA256: digest(buf.Bytes()),
|
|
}, nil
|
|
}
|
|
|
|
func digest(data []byte) string {
|
|
sum := sha256.Sum256(data)
|
|
return hex.EncodeToString(sum[:])
|
|
}
|