feat: add durable image processing pipeline

This commit is contained in:
2026-10-22 17:22:28 +03:00
parent 8c2ac690a6
commit 42336ce7c2
39 changed files with 4379 additions and 126 deletions
+10 -1
View File
@@ -24,8 +24,17 @@ func run(logger *slog.Logger, arguments []string) error {
if len(arguments) == 1 && arguments[0] == "migrate" {
return bootstrap.RunMigrations()
}
if len(arguments) == 1 && arguments[0] == "dispatch" {
return bootstrap.RunDispatcher(logger)
}
if len(arguments) == 1 && arguments[0] == "worker" {
return bootstrap.RunWorker(logger)
}
if arguments[0] == "reconcile-attachments" {
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
}
return errors.New("usage: messenger [migrate|reconcile-attachments]")
if arguments[0] == "replay-attachment" {
return bootstrap.RunAttachmentReplay(arguments[1:], os.Stdout)
}
return errors.New("usage: messenger [migrate|dispatch|worker|reconcile-attachments|replay-attachment]")
}
+3
View File
@@ -3,10 +3,12 @@ module gitea.privcloud.tech/dizi/mtusi2026lab5
go 1.26.0
require (
github.com/deepteams/webp v1.2.8
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
github.com/minio/minio-go/v7 v7.0.99
github.com/prometheus/client_golang v1.23.2
github.com/redis/go-redis/v9 v9.17.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/valkey-io/valkey-go v1.0.78
@@ -18,6 +20,7 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
+10
View File
@@ -1,10 +1,18 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deepteams/webp v1.2.8 h1:iFLBxo4V34ixqay6+MTB18zujCM8VSe1ejsYh+ue1nA=
github.com/deepteams/webp v1.2.8/go.mod h1:J8Ap+HAixxpKKRN9IpEeSKlfvhsef1v43jKTO7m3f4c=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -64,6 +72,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+200
View File
@@ -0,0 +1,200 @@
package bootstrap
import (
"context"
"errors"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
)
func RunDispatcher(logger *slog.Logger) error {
settings, err := config.LoadDispatcher()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
client, err := newRedisClient(settings.BrokerURL)
if err != nil {
return err
}
defer func() { _ = client.Close() }()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher",
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength,
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize),
})
if err != nil {
return err
}
if err := broker.Ensure(context.Background()); err != nil {
return err
}
metrics := pipeline.NewMetrics()
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
RetryDelay: settings.RetryDelay,
})
if err != nil {
return err
}
logger.Info("outbox dispatcher started", "stream", settings.Stream)
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run)
}
func RunWorker(logger *slog.Logger) error {
settings, err := config.LoadWorker()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
client, err := newRedisClient(settings.BrokerURL)
if err != nil {
return err
}
defer func() { _ = client.Close() }()
hostname, err := os.Hostname()
if err != nil {
return err
}
consumer := hostname + "-" + uuid.NewString()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock,
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch),
})
if err != nil {
return err
}
if err := broker.Ensure(context.Background()); err != nil {
return err
}
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
if err != nil {
return err
}
metrics := pipeline.NewMetrics()
processorWorker, err := worker.New(
logger, postgres.New(pool), broker, objects, processor, metrics,
worker.Config{
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
ProcessingTimeout: settings.ProcessingTimeout,
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
MemoryBytes: settings.MemoryBytes,
},
)
if err != nil {
return err
}
logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency)
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run)
}
func newRedisClient(rawURL string) (*redis.Client, error) {
options, err := redis.ParseURL(rawURL)
if err != nil {
return nil, err
}
client := redis.NewClient(options)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := client.Ping(ctx).Err(); err != nil {
_ = client.Close()
return nil, err
}
return client, nil
}
func runPipelineProcess(
logger *slog.Logger,
metricsAddress string,
metricsHandler http.Handler,
run func(context.Context) error,
) error {
mux := http.NewServeMux()
mux.Handle("/metrics", metricsHandler)
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
listener, err := net.Listen("tcp", metricsAddress)
if err != nil {
return err
}
serveResult := make(chan error, 1)
go func() { serveResult <- server.Serve(listener) }()
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
runResult := make(chan error, 1)
go func() { runResult <- run(ctx) }()
var result error
select {
case result = <-runResult:
stop()
case result = <-serveResult:
stop()
case <-ctx.Done():
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil && result == nil {
result = err
}
if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) {
return nil
}
logger.Info("pipeline process stopped")
return result
}
func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config {
return database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
}
}
func workerDatabaseConfig(settings config.WorkerConfig) database.Config {
return database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
}
}
+1
View File
@@ -64,6 +64,7 @@ func RunReconciliation(arguments []string, output io.Writer) error {
}
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod,
OrphanPrefixes: []string{"staging/"},
UnreferencedRetention: *retention, Apply: *apply,
})
if err != nil {
+39
View File
@@ -0,0 +1,39 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"io"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
)
func RunAttachmentReplay(arguments []string, output io.Writer) error {
if len(arguments) != 3 {
return errors.New("usage: messenger replay-attachment <attachment-id> <actor> <reason>")
}
settings, err := config.LoadJobAdmin()
if err != nil {
return err
}
pool, err := database.Open(context.Background(), database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
job, err := postgres.New(pool).ReplayFailedAttachment(
context.Background(), arguments[0], arguments[1], arguments[2],
)
if err != nil {
return err
}
_, err = fmt.Fprintf(output, "replayed attachment=%s job=%s attempt=%d\n", job.AttachmentID, job.ID, job.Attempt)
return err
}
+1 -1
View File
@@ -54,7 +54,7 @@ func RunServer(logger *slog.Logger) error {
attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory,
CleanupTimeout: settings.RequestTimeout,
CleanupTimeout: settings.RequestTimeout, PipelineVersion: settings.PipelineVersion,
})
if err != nil {
return err
+280
View File
@@ -0,0 +1,280 @@
package redisstream
import (
"context"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
)
const payloadField = "payload"
var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
var ErrDeliveryNotOwned = errors.New("redis stream delivery is not owned by this consumer")
var _ outbox.Publisher = (*Broker)(nil)
type Config struct {
Stream string
Group string
Consumer string
FailedStream string
FailedMaxLen int64
Block time.Duration
ClaimMinIdle time.Duration
ReadCount int64
}
type Broker struct {
client redis.UniversalClient
config Config
}
type Delivery struct {
ID string
Envelope Envelope
DecodeError error
broker *Broker
payload []byte
}
type FailedDelivery struct {
Code string
FailedAt time.Time
}
func New(client redis.UniversalClient, config Config) (*Broker, error) {
if client == nil {
return nil, errors.New("redis client is required")
}
if err := config.validate(); err != nil {
return nil, err
}
return &Broker{client: client, config: config}, nil
}
func (c Config) validate() error {
switch {
case c.Stream == "":
return errors.New("stream is required")
case c.Group == "":
return errors.New("consumer group is required")
case c.Consumer == "":
return errors.New("consumer name is required")
case c.FailedStream == "":
return errors.New("failed stream is required")
case c.FailedStream == c.Stream:
return errors.New("failed stream must differ from job stream")
case c.FailedMaxLen <= 0:
return errors.New("failed stream maximum length must be positive")
case c.Block < 0:
return errors.New("block duration must not be negative")
case c.ClaimMinIdle <= 0:
return errors.New("claim minimum idle time must be positive")
case c.ReadCount <= 0:
return errors.New("read count must be positive")
default:
return nil
}
}
func (b *Broker) Ensure(ctx context.Context) error {
err := b.client.XGroupCreateMkStream(ctx, b.config.Stream, b.config.Group, "0").Err()
if err != nil && !isBusyGroup(err) {
return fmt.Errorf("create redis stream consumer group: %w", err)
}
return nil
}
func isBusyGroup(err error) bool {
return err != nil && strings.HasPrefix(err.Error(), "BUSYGROUP")
}
func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) {
envelope, err := DecodeEnvelope(payload)
if err != nil {
return "", err
}
payload, err = EncodeEnvelope(envelope)
if err != nil {
return "", err
}
id, err := b.client.XAdd(ctx, &redis.XAddArgs{
Stream: b.config.Stream,
Values: map[string]any{payloadField: string(payload)},
}).Result()
if err != nil {
return "", fmt.Errorf("publish redis stream job: %w", err)
}
return id, nil
}
func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: b.config.Group,
Consumer: b.config.Consumer,
Streams: []string{b.config.Stream, ">"},
Count: b.config.ReadCount,
Block: b.config.Block,
}).Result()
if errors.Is(err, redis.Nil) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("read redis stream jobs: %w", err)
}
return b.deliveries(streams)
}
func (b *Broker) Reclaim(ctx context.Context, start string) ([]Delivery, string, error) {
messages, next, err := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{
Stream: b.config.Stream,
Group: b.config.Group,
Consumer: b.config.Consumer,
MinIdle: b.config.ClaimMinIdle,
Start: start,
Count: b.config.ReadCount,
}).Result()
if err != nil {
return nil, start, fmt.Errorf("reclaim redis stream jobs: %w", err)
}
deliveries, err := b.deliveries([]redis.XStream{{Stream: b.config.Stream, Messages: messages}})
if err != nil {
return nil, start, err
}
return deliveries, next, nil
}
func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) {
var deliveries []Delivery
for _, stream := range streams {
for _, message := range stream.Messages {
delivery := Delivery{ID: message.ID, broker: b}
payload, err := messagePayload(message)
if err != nil {
delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err)
deliveries = append(deliveries, delivery)
continue
}
envelope, err := DecodeEnvelope(payload)
if err != nil {
delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err)
deliveries = append(deliveries, delivery)
continue
}
delivery.Envelope = envelope
delivery.payload, err = EncodeEnvelope(envelope)
if err != nil {
return nil, fmt.Errorf("encode validated redis stream message %s: %w", message.ID, err)
}
deliveries = append(deliveries, delivery)
}
}
return deliveries, nil
}
func messagePayload(message redis.XMessage) ([]byte, error) {
value, ok := message.Values[payloadField]
if !ok {
return nil, errors.New("payload field is missing")
}
switch payload := value.(type) {
case string:
return []byte(payload), nil
case []byte:
return payload, nil
default:
return nil, fmt.Errorf("payload has type %T", value)
}
}
func (d Delivery) Ack(ctx context.Context) error {
if d.broker == nil {
return errors.New("delivery is detached from broker")
}
acked, err := d.broker.client.Eval(ctx, ackOwnedScript, []string{d.broker.config.Stream},
d.broker.config.Group, d.ID, d.broker.config.Consumer,
).Int64()
if err != nil {
return fmt.Errorf("ack redis stream job: %w", err)
}
if acked == 0 {
return ErrDeliveryNotOwned
}
return nil
}
const ackOwnedScript = `
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
if #pending == 0 or pending[1][2] ~= ARGV[3] then
return 0
end
local acked = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
if acked == 1 then
redis.call('XDEL', KEYS[1], ARGV[2])
end
return acked
`
func (d Delivery) ToFailed(ctx context.Context, failure FailedDelivery) error {
if d.broker == nil {
return errors.New("delivery is detached from broker")
}
if !failureCodePattern.MatchString(failure.Code) {
return errors.New("failure code is invalid")
}
if failure.FailedAt.IsZero() {
return errors.New("failure time is required")
}
payload := d.payload
if d.DecodeError == nil && len(payload) == 0 {
var err error
payload, err = EncodeEnvelope(d.Envelope)
if err != nil {
return err
}
}
moved, err := d.broker.client.Eval(ctx, moveToFailedScript,
[]string{d.broker.config.FailedStream, d.broker.config.Stream},
d.broker.config.Group, d.ID, d.broker.config.Consumer, string(payload), failure.Code,
failure.FailedAt.UTC().Format(time.RFC3339Nano), strconv.Itoa(d.Envelope.Attempt),
strconv.FormatInt(d.broker.config.FailedMaxLen, 10),
).Int64()
if err != nil {
return fmt.Errorf("move redis stream job to failed stream: %w", err)
}
if moved == 0 {
return ErrDeliveryNotOwned
}
return nil
}
const moveToFailedScript = `
local pending = redis.call('XPENDING', KEYS[2], ARGV[1], ARGV[2], ARGV[2], 1)
if #pending == 0 or pending[1][2] ~= ARGV[3] then
return 0
end
redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[8], '*',
'payload', ARGV[4],
'source_message_id', ARGV[2],
'error_code', ARGV[5],
'failed_at', ARGV[6],
'attempt', ARGV[7])
local acked = redis.call('XACK', KEYS[2], ARGV[1], ARGV[2])
if acked == 1 then
redis.call('XDEL', KEYS[2], ARGV[2])
end
return acked
`
@@ -0,0 +1,74 @@
package redisstream
import (
"testing"
"time"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
func TestBrokerDecodesDeliveries(t *testing.T) {
payload, err := EncodeEnvelope(validEnvelope())
require.NoError(t, err)
broker := &Broker{config: Config{Stream: "attachment-jobs"}}
deliveries, err := broker.deliveries([]redis.XStream{{
Stream: "attachment-jobs",
Messages: []redis.XMessage{{
ID: "1-0",
Values: map[string]any{payloadField: string(payload)},
}},
}})
require.NoError(t, err)
require.Len(t, deliveries, 1)
require.Equal(t, "1-0", deliveries[0].ID)
require.Equal(t, validEnvelope(), deliveries[0].Envelope)
}
func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) {
broker := &Broker{config: Config{Stream: "attachment-jobs"}}
deliveries, err := broker.deliveries([]redis.XStream{{
Stream: "attachment-jobs",
Messages: []redis.XMessage{{
ID: "1-0",
Values: map[string]any{payloadField: `{
"schema_version":1,
"job_id":"11111111-1111-4111-8111-111111111111",
"type":"attachment.process.v1",
"attachment_id":"22222222-2222-4222-8222-222222222222",
"pipeline_version":1,
"attempt":1,
"created_at":"2026-09-01T12:00:00Z",
"storage_key":"private/object"
}`},
}},
}})
require.NoError(t, err)
require.Len(t, deliveries, 1)
require.Error(t, deliveries[0].DecodeError)
require.Empty(t, deliveries[0].payload)
}
func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) {
broker := &Broker{}
_, err := broker.PublishConfirmed(t.Context(), []byte(`{"schema_version":2}`))
require.Error(t, err)
}
func TestConfigValidation(t *testing.T) {
config := Config{
Stream: "attachment-jobs",
Group: "attachment-workers",
Consumer: "worker-1",
FailedStream: "attachment-jobs-failed",
FailedMaxLen: 1000,
Block: time.Second,
ClaimMinIdle: time.Minute,
ReadCount: 10,
}
require.NoError(t, config.validate())
config.ClaimMinIdle = 0
require.Error(t, config.validate())
}
+147
View File
@@ -0,0 +1,147 @@
package redisstream
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"regexp"
"time"
"unicode/utf8"
"github.com/google/uuid"
)
const (
SchemaVersion = 1
JobType = "attachment.process.v1"
)
var (
errInvalidEnvelope = errors.New("invalid job envelope")
traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
)
type Envelope struct {
SchemaVersion int `json:"schema_version"`
JobID uuid.UUID `json:"job_id"`
Type string `json:"type"`
AttachmentID uuid.UUID `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
}
func (e Envelope) Validate() error {
switch {
case e.SchemaVersion != SchemaVersion:
return fmt.Errorf("%w: unsupported schema_version", errInvalidEnvelope)
case e.Type != JobType:
return fmt.Errorf("%w: unsupported type", errInvalidEnvelope)
case e.JobID == uuid.Nil:
return fmt.Errorf("%w: job_id is required", errInvalidEnvelope)
case e.AttachmentID == uuid.Nil:
return fmt.Errorf("%w: attachment_id is required", errInvalidEnvelope)
case e.PipelineVersion < 1:
return fmt.Errorf("%w: pipeline_version must be positive", errInvalidEnvelope)
case e.Attempt < 1:
return fmt.Errorf("%w: attempt must be positive", errInvalidEnvelope)
case e.CreatedAt.IsZero():
return fmt.Errorf("%w: created_at is required", errInvalidEnvelope)
case e.Traceparent != "" && !traceparentPattern.MatchString(e.Traceparent):
return fmt.Errorf("%w: invalid traceparent", errInvalidEnvelope)
case utf8.RuneCountInString(e.Tracestate) > 512:
return fmt.Errorf("%w: tracestate is too long", errInvalidEnvelope)
default:
return nil
}
}
func EncodeEnvelope(envelope Envelope) ([]byte, error) {
if err := envelope.Validate(); err != nil {
return nil, err
}
payload, err := json.Marshal(envelope)
if err != nil {
return nil, fmt.Errorf("encode job envelope: %w", err)
}
return payload, nil
}
func DecodeEnvelope(payload []byte) (Envelope, error) {
var wire envelopeJSON
decoder := json.NewDecoder(bytes.NewReader(payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&wire); err != nil {
return Envelope{}, fmt.Errorf("%w: decode: %v", errInvalidEnvelope, err)
}
if err := ensureJSONEnd(decoder); err != nil {
return Envelope{}, err
}
jobID, err := parseUUID(wire.JobID, "job_id")
if err != nil {
return Envelope{}, err
}
attachmentID, err := parseUUID(wire.AttachmentID, "attachment_id")
if err != nil {
return Envelope{}, err
}
createdAt, err := time.Parse(time.RFC3339, wire.CreatedAt)
if err != nil {
return Envelope{}, fmt.Errorf("%w: invalid created_at", errInvalidEnvelope)
}
envelope := Envelope{
SchemaVersion: wire.SchemaVersion,
JobID: jobID,
Type: wire.Type,
AttachmentID: attachmentID,
PipelineVersion: wire.PipelineVersion,
Attempt: wire.Attempt,
CreatedAt: createdAt,
Traceparent: wire.Traceparent,
Tracestate: wire.Tracestate,
}
if err := envelope.Validate(); err != nil {
return Envelope{}, err
}
return envelope, nil
}
type envelopeJSON struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
}
func parseUUID(value, field string) (uuid.UUID, error) {
if !uuidPattern.MatchString(value) {
return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field)
}
parsed, err := uuid.Parse(value)
if err != nil {
return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field)
}
return parsed, nil
}
func ensureJSONEnd(decoder *json.Decoder) error {
var value any
if err := decoder.Decode(&value); !errors.Is(err, io.EOF) {
return fmt.Errorf("%w: trailing JSON value", errInvalidEnvelope)
}
return nil
}
@@ -0,0 +1,74 @@
package redisstream
import (
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
)
func TestEnvelopeRoundTrip(t *testing.T) {
envelope := validEnvelope()
payload, err := EncodeEnvelope(envelope)
require.NoError(t, err)
decoded, err := DecodeEnvelope(payload)
require.NoError(t, err)
require.Equal(t, envelope, decoded)
}
func TestDecodeEnvelopeRejectsUnknownField(t *testing.T) {
payload := []byte(`{
"schema_version":1,
"job_id":"11111111-1111-4111-8111-111111111111",
"type":"attachment.process.v1",
"attachment_id":"22222222-2222-4222-8222-222222222222",
"pipeline_version":1,
"attempt":1,
"created_at":"2026-09-01T12:00:00Z",
"storage_key":"must-not-enter-the-broker"
}`)
_, err := DecodeEnvelope(payload)
require.Error(t, err)
}
func TestEnvelopeValidation(t *testing.T) {
tests := []struct {
name string
mutate func(*Envelope)
}{
{name: "schema", mutate: func(e *Envelope) { e.SchemaVersion = 2 }},
{name: "type", mutate: func(e *Envelope) { e.Type = "attachment.delete.v1" }},
{name: "job id", mutate: func(e *Envelope) { e.JobID = uuid.Nil }},
{name: "attachment id", mutate: func(e *Envelope) { e.AttachmentID = uuid.Nil }},
{name: "pipeline", mutate: func(e *Envelope) { e.PipelineVersion = 0 }},
{name: "attempt", mutate: func(e *Envelope) { e.Attempt = 0 }},
{name: "created at", mutate: func(e *Envelope) { e.CreatedAt = time.Time{} }},
{name: "traceparent", mutate: func(e *Envelope) { e.Traceparent = "invalid" }},
{name: "tracestate length", mutate: func(e *Envelope) { e.Tracestate = string(make([]byte, 513)) }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
envelope := validEnvelope()
tt.mutate(&envelope)
require.Error(t, envelope.Validate())
})
}
}
func validEnvelope() Envelope {
return Envelope{
SchemaVersion: SchemaVersion,
JobID: uuid.MustParse("11111111-1111-4111-8111-111111111111"),
Type: JobType,
AttachmentID: uuid.MustParse("22222222-2222-4222-8222-222222222222"),
PipelineVersion: 1,
Attempt: 1,
CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC),
Traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
Tracestate: "vendor=value",
}
}
+6
View File
@@ -46,6 +46,7 @@ type ServerConfig struct {
AttachmentMaxPixels int64
AttachmentBodyMemory int64
AttachmentDecodeMemory int64
PipelineVersion int
}
type MigrationConfig struct {
@@ -207,6 +208,10 @@ func LoadServer() (ServerConfig, error) {
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
}
pipelineVersion, err := integer(values, "PIPELINE_VERSION", 1, 1000)
if err != nil {
return ServerConfig{}, err
}
return ServerConfig{
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
@@ -226,6 +231,7 @@ func LoadServer() (ServerConfig, error) {
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
AttachmentBodyMemory: int64(attachmentBodyMemory),
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
PipelineVersion: pipelineVersion,
}, nil
}
+273
View File
@@ -0,0 +1,273 @@
package config
import (
"errors"
"net"
"time"
"github.com/spf13/viper"
)
type DispatcherConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
BrokerURL string
Stream string
ConsumerGroup string
FailedStream string
FailedStreamMaxLength int64
BatchSize int
LeaseDuration time.Duration
PollInterval time.Duration
RecoveryInterval time.Duration
RetryDelay time.Duration
MetricsAddress string
}
type WorkerConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
S3UseTLS bool
S3Timeout time.Duration
BrokerURL string
Stream string
ConsumerGroup string
FailedStream string
FailedStreamMaxLength int64
Concurrency int
Prefetch int
LeaseDuration time.Duration
ProcessingTimeout time.Duration
ReadBlock time.Duration
ReclaimInterval time.Duration
MaxAttempts int
RetryBase time.Duration
RetryMax time.Duration
AttachmentMaxBytes int64
AttachmentMaxPixels int64
MemoryBytes int64
MetricsAddress string
}
type JobAdminConfig struct {
DatabaseURL string
DBMaxConns int32
DBMinConns int32
DBMaxConnectionAge time.Duration
DBMaxConnectionIdle time.Duration
DBHealthCheckPeriod time.Duration
DependencyTimeout time.Duration
}
type brokerConfig struct {
url string
stream string
consumerGroup string
failedStream string
failedStreamMaxLength int
}
func LoadJobAdmin() (JobAdminConfig, error) {
values := viper.New()
values.AutomaticEnv()
database, err := loadDatabaseConfig(values)
if err != nil {
return JobAdminConfig{}, err
}
return JobAdminConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
}, nil
}
func LoadDispatcher() (DispatcherConfig, error) {
values := viper.New()
values.AutomaticEnv()
database, err := loadDatabaseConfig(values)
if err != nil {
return DispatcherConfig{}, err
}
broker, err := loadBrokerConfig(values)
if err != nil {
return DispatcherConfig{}, err
}
batchSize, err := integer(values, "OUTBOX_BATCH_SIZE", 1, 1000)
if err != nil {
return DispatcherConfig{}, err
}
leaseDuration, err := seconds(values, "OUTBOX_LEASE_SECONDS", 1, 600)
if err != nil {
return DispatcherConfig{}, err
}
pollInterval, err := seconds(values, "OUTBOX_POLL_SECONDS", 1, 60)
if err != nil {
return DispatcherConfig{}, err
}
recoveryInterval, err := seconds(values, "OUTBOX_RECOVERY_SECONDS", 1, 3600)
if err != nil {
return DispatcherConfig{}, err
}
retryDelay, err := seconds(values, "OUTBOX_RETRY_SECONDS", 1, 3600)
if err != nil {
return DispatcherConfig{}, err
}
metricsAddress, err := metricsAddress(values)
if err != nil {
return DispatcherConfig{}, err
}
return DispatcherConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup,
FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval,
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, MetricsAddress: metricsAddress,
}, nil
}
func LoadWorker() (WorkerConfig, error) {
values := viper.New()
values.AutomaticEnv()
database, err := loadDatabaseConfig(values)
if err != nil {
return WorkerConfig{}, err
}
storage, err := loadObjectStorageConfig(values)
if err != nil {
return WorkerConfig{}, err
}
broker, err := loadBrokerConfig(values)
if err != nil {
return WorkerConfig{}, err
}
concurrency, err := integer(values, "WORKER_CONCURRENCY", 1, 64)
if err != nil {
return WorkerConfig{}, err
}
prefetch, err := integer(values, "WORKER_PREFETCH", concurrency, 1024)
if err != nil {
return WorkerConfig{}, err
}
leaseDuration, err := seconds(values, "WORKER_LEASE_SECONDS", 1, 3600)
if err != nil {
return WorkerConfig{}, err
}
processingTimeout, err := seconds(values, "WORKER_PROCESSING_TIMEOUT_SECONDS", 1, 3600)
if err != nil {
return WorkerConfig{}, err
}
if processingTimeout >= leaseDuration {
return WorkerConfig{}, errors.New("WORKER_PROCESSING_TIMEOUT_SECONDS must be less than WORKER_LEASE_SECONDS")
}
readBlock, err := seconds(values, "WORKER_READ_BLOCK_SECONDS", 1, 60)
if err != nil {
return WorkerConfig{}, err
}
reclaimInterval, err := seconds(values, "WORKER_RECLAIM_SECONDS", 1, 3600)
if err != nil {
return WorkerConfig{}, err
}
maxAttempts, err := integer(values, "WORKER_MAX_ATTEMPTS", 1, 100)
if err != nil {
return WorkerConfig{}, err
}
retryBase, err := seconds(values, "WORKER_RETRY_BASE_SECONDS", 1, 3600)
if err != nil {
return WorkerConfig{}, err
}
retryMax, err := seconds(values, "WORKER_RETRY_MAX_SECONDS", 1, 86400)
if err != nil {
return WorkerConfig{}, err
}
if retryMax < retryBase {
return WorkerConfig{}, errors.New("WORKER_RETRY_MAX_SECONDS must be at least WORKER_RETRY_BASE_SECONDS")
}
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
if err != nil {
return WorkerConfig{}, err
}
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
if err != nil {
return WorkerConfig{}, err
}
memoryBytes, err := integer(values, "WORKER_MEMORY_BYTES", attachmentMaxBytes, 2*1024*1024*1024)
if err != nil {
return WorkerConfig{}, err
}
metricsAddress, err := metricsAddress(values)
if err != nil {
return WorkerConfig{}, err
}
return WorkerConfig{
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS,
S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream,
ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream,
FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration,
ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax,
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
MemoryBytes: int64(memoryBytes),
MetricsAddress: metricsAddress,
}, nil
}
func metricsAddress(values *viper.Viper) (string, error) {
host, err := requiredString(values, "PIPELINE_METRICS_HOST")
if err != nil {
return "", err
}
port, err := requiredPort(values, "PIPELINE_METRICS_PORT")
if err != nil {
return "", err
}
return net.JoinHostPort(host, port), nil
}
func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) {
url, err := requiredString(values, "BROKER_URL")
if err != nil {
return brokerConfig{}, err
}
stream, err := requiredString(values, "BROKER_STREAM")
if err != nil {
return brokerConfig{}, err
}
consumerGroup, err := requiredString(values, "BROKER_CONSUMER_GROUP")
if err != nil {
return brokerConfig{}, err
}
failedStream, err := requiredString(values, "BROKER_FAILED_STREAM")
if err != nil {
return brokerConfig{}, err
}
failedStreamMaxLength, err := integer(values, "BROKER_FAILED_MAX_LENGTH", 100, 10_000_000)
if err != nil {
return brokerConfig{}, err
}
return brokerConfig{
url: url, stream: stream, consumerGroup: consumerGroup,
failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength,
}, nil
}
@@ -0,0 +1,102 @@
package pipeline
import (
"net/http"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
type Metrics struct {
workerInFlight prometheus.Gauge
workerJobs *prometheus.CounterVec
workerDuration *prometheus.HistogramVec
workerRetries prometheus.Counter
workerReclaimed prometheus.Counter
outboxPublished prometheus.Counter
outboxPublishFailure prometheus.Counter
outboxRecovered prometheus.Counter
handler http.Handler
}
func NewMetrics() *Metrics {
registry := prometheus.NewRegistry()
metrics := &Metrics{
workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight",
Help: "Attachment jobs currently processed by this worker.",
}),
workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total",
Help: "Completed attachment job deliveries by result.",
}, []string{"result"}),
workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds",
Help: "Attachment job delivery processing duration.", Buckets: prometheus.DefBuckets,
}, []string{"result"}),
workerRetries: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total",
Help: "Attachment jobs scheduled for another attempt.",
}),
workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total",
Help: "Redis Stream deliveries reclaimed after their idle timeout.",
}),
outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "published_total",
Help: "Outbox events confirmed by Redis Streams.",
}),
outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total",
Help: "Outbox events whose Redis Streams publish failed.",
}),
outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total",
Help: "Expired outbox leases returned for publication.",
}),
}
registry.MustRegister(
metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration,
metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished,
metrics.outboxPublishFailure, metrics.outboxRecovered,
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
)
metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
return metrics
}
func (m *Metrics) Handler() http.Handler {
return m.handler
}
func (m *Metrics) ObserveStarted() {
m.workerInFlight.Inc()
}
func (m *Metrics) ObserveFinished(result string, duration time.Duration) {
m.workerInFlight.Dec()
m.workerJobs.WithLabelValues(result).Inc()
m.workerDuration.WithLabelValues(result).Observe(duration.Seconds())
}
func (m *Metrics) ObserveRetry() {
m.workerRetries.Inc()
}
func (m *Metrics) ObserveReclaimed(count int) {
m.workerReclaimed.Add(float64(count))
}
func (m *Metrics) ObservePublished() {
m.outboxPublished.Inc()
}
func (m *Metrics) ObservePublishFailure() {
m.outboxPublishFailure.Inc()
}
func (m *Metrics) ObserveRecovered(count int) {
m.outboxRecovered.Add(float64(count))
}
+29 -45
View File
@@ -25,10 +25,10 @@ import (
type Repository interface {
ChatForUser(context.Context, string, string) (model.Chat, error)
CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error)
MarkAttachmentReady(context.Context, string, string) (model.Attachment, error)
QueueAttachmentWithJobAndOutbox(context.Context, string, string, model.AttachmentJob, string, string) (model.Attachment, error)
DeletePendingAttachment(context.Context, string, string) error
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error)
}
type ObjectStore interface {
@@ -43,6 +43,7 @@ type Config struct {
BodyMemoryBytes int64
DecodeMemoryBytes int64
CleanupTimeout time.Duration
PipelineVersion int
}
type Upload struct {
@@ -52,10 +53,13 @@ type Upload struct {
ContentType string
ContentLength int64
Body io.Reader
Traceparent string
Tracestate string
}
type Content struct {
Attachment model.Attachment
Variant model.Variant
Body io.ReadCloser
}
@@ -69,7 +73,7 @@ type Service struct {
}
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 || config.PipelineVersion < 1 {
return nil, errors.New("invalid attachment service configuration")
}
if config.BodyMemoryBytes < config.MaxBytes {
@@ -96,58 +100,38 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e
defer release()
attachmentID := uuid.NewString()
objectKey := "attachments/" + uuid.NewString() + "/original"
stagingKey := "staging/" + attachmentID + "/" + uuid.NewString()
digest := sha256.Sum256(data)
pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{
ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename,
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
ObjectKey: objectKey, Status: model.AttachmentPending,
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending,
PipelineVersion: s.config.PipelineVersion,
})
if err != nil {
return model.Attachment{}, mapDependencyError(err)
}
if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil {
s.deletePending(pending.ID, objectKey)
if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil {
s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err)
s.deletePending(pending.ID, stagingKey)
return model.Attachment{}, model.ErrUnavailable
}
attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey)
if err == nil {
return attachment, nil
createdAt := time.Now().UTC()
queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{
ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion,
Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt,
}, input.Traceparent, input.Tracestate)
if err != nil {
s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
return model.Attachment{}, mapDependencyError(err)
}
return s.resolveReadyOutcome(pending.ID, objectKey)
return queued, nil
}
func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) {
func (s *Service) deletePending(attachmentID, stagingKey string) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
defer cancel()
stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
if err == nil && stored.Status == model.AttachmentReady {
return stored, nil
}
if err == nil && stored.Status == model.AttachmentPending {
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr)
return model.Attachment{}, model.ErrUnavailable
}
if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr)
}
return model.Attachment{}, model.ErrUnavailable
}
if errors.Is(err, model.ErrNotFound) {
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr)
}
return model.Attachment{}, model.ErrUnavailable
}
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err)
return model.Attachment{}, model.ErrUnavailable
}
func (s *Service) deletePending(attachmentID, objectKey string) {
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
defer cancel()
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil {
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil {
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
}
}
@@ -160,16 +144,16 @@ func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (mo
return attachment, nil
}
func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) {
attachment, err := s.Metadata(ctx, attachmentID, userID)
func (s *Service) Content(ctx context.Context, attachmentID, userID, variantName string) (Content, error) {
attachment, variant, err := s.repository.AttachmentVariantForUser(ctx, attachmentID, userID, variantName)
if err != nil {
return Content{}, err
return Content{}, mapDependencyError(err)
}
body, err := s.objects.Get(ctx, attachment.ObjectKey)
body, err := s.objects.Get(ctx, variant.ObjectKey)
if err != nil {
return Content{}, model.ErrUnavailable
}
return Content{Attachment: attachment, Body: body}, nil
return Content{Attachment: attachment, Variant: variant, Body: body}, nil
}
func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
+26 -24
View File
@@ -27,8 +27,8 @@ func TestUploadLifecycle(t *testing.T) {
result, err := service.Upload(context.Background(), upload(data, "image/png"))
require.NoError(t, err)
require.Equal(t, model.AttachmentReady, result.Status)
require.Equal(t, []string{"pending", "put", "ready"}, events)
require.Equal(t, model.AttachmentQueued, result.Status)
require.Equal(t, []string{"pending", "put", "queued"}, events)
}
func TestUploadValidation(t *testing.T) {
@@ -86,7 +86,7 @@ func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
Config{
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, PipelineVersion: 1,
},
)
require.Error(t, err)
@@ -103,16 +103,16 @@ func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
require.True(t, repository.pendingDeleted)
}
func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) {
func TestUploadKeepsStagingAfterQueueFailure(t *testing.T) {
data := pngFixture(t, 2, 2)
repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true}
repository := &repositoryStub{queueErr: errors.New("connection lost")}
objects := &objectStub{}
service := newTestService(t, repository, objects, 1024, 4)
result, err := service.Upload(context.Background(), upload(data, "image/png"))
require.NoError(t, err)
require.Equal(t, model.AttachmentReady, result.Status)
_, err := service.Upload(context.Background(), upload(data, "image/png"))
require.ErrorIs(t, err, model.ErrUnavailable)
require.False(t, objects.deleted)
require.False(t, repository.pendingDeleted)
}
func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service {
@@ -121,7 +121,7 @@ func newTestService(t *testing.T, repository Repository, objects ObjectStore, ma
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects,
Config{
MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes,
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second,
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, PipelineVersion: 1,
},
)
require.NoError(t, err)
@@ -137,8 +137,7 @@ func upload(data []byte, contentType string) Upload {
type repositoryStub struct {
events *[]string
markErr error
lookupReady bool
queueErr error
pendingDeleted bool
pending model.Attachment
}
@@ -154,12 +153,19 @@ func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment m
return attachment, nil
}
func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) {
if s.markErr != nil {
return model.Attachment{}, s.markErr
func (s *repositoryStub) QueueAttachmentWithJobAndOutbox(
context.Context,
string,
string,
model.AttachmentJob,
string,
string,
) (model.Attachment, error) {
if s.queueErr != nil {
return model.Attachment{}, s.queueErr
}
s.pending.Status = model.AttachmentReady
s.record("ready")
s.pending.Status = model.AttachmentQueued
s.record("queued")
return s.pending, nil
}
@@ -168,18 +174,14 @@ func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string
return nil
}
func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
if s.lookupReady {
s.pending.Status = model.AttachmentReady
return s.pending, nil
}
return model.Attachment{}, model.ErrNotFound
}
func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
return s.pending, nil
}
func (s *repositoryStub) AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) {
return s.pending, model.Variant{}, nil
}
func (s *repositoryStub) record(event string) {
if s.events != nil {
*s.events = append(*s.events, event)
@@ -0,0 +1,146 @@
package transform
import (
"encoding/binary"
"errors"
"image"
"image/color"
)
var errInvalidEXIF = errors.New("invalid EXIF metadata")
func jpegOrientation(data []byte) (int, error) {
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
return 0, errInvalidEXIF
}
for offset := 2; offset < len(data); {
if data[offset] != 0xff {
return 0, errInvalidEXIF
}
for offset < len(data) && data[offset] == 0xff {
offset++
}
if offset >= len(data) {
return 0, errInvalidEXIF
}
marker := data[offset]
offset++
if marker == 0xd9 || marker == 0xda {
return 1, nil
}
if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 {
continue
}
if offset+2 > len(data) {
return 0, errInvalidEXIF
}
length := int(binary.BigEndian.Uint16(data[offset : offset+2]))
if length < 2 || offset+length > len(data) {
return 0, errInvalidEXIF
}
payload := data[offset+2 : offset+length]
offset += length
if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" {
continue
}
return tiffOrientation(payload[6:])
}
return 1, nil
}
func tiffOrientation(data []byte) (int, error) {
if len(data) < 8 {
return 0, errInvalidEXIF
}
var order binary.ByteOrder
switch string(data[:2]) {
case "II":
order = binary.LittleEndian
case "MM":
order = binary.BigEndian
default:
return 0, errInvalidEXIF
}
if order.Uint16(data[2:4]) != 42 {
return 0, errInvalidEXIF
}
ifdOffset := uint64(order.Uint32(data[4:8]))
if ifdOffset > uint64(len(data)-2) {
return 0, errInvalidEXIF
}
count := uint64(order.Uint16(data[ifdOffset : ifdOffset+2]))
entriesStart := ifdOffset + 2
if count > (uint64(len(data))-entriesStart)/12 {
return 0, errInvalidEXIF
}
for index := uint64(0); index < count; index++ {
entry := data[entriesStart+index*12 : entriesStart+(index+1)*12]
if order.Uint16(entry[:2]) != 0x0112 {
continue
}
if order.Uint16(entry[2:4]) != 3 || order.Uint32(entry[4:8]) != 1 {
return 0, errInvalidEXIF
}
orientation := int(order.Uint16(entry[8:10]))
if orientation < 1 || orientation > 8 {
return 0, errInvalidEXIF
}
return orientation, nil
}
return 1, nil
}
func applyOrientation(source image.Image, orientation int) *image.NRGBA {
bounds := source.Bounds()
width, height := bounds.Dx(), bounds.Dy()
outputWidth, outputHeight := width, height
if orientation >= 5 {
outputWidth, outputHeight = height, width
}
output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight))
for y := 0; y < outputHeight; y++ {
for x := 0; x < outputWidth; x++ {
sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation)
output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY))
}
}
return output
}
func orientedSourcePoint(x, y, width, height, orientation int) (int, int) {
switch orientation {
case 2:
return width - 1 - x, y
case 3:
return width - 1 - x, height - 1 - y
case 4:
return x, height - 1 - y
case 5:
return y, x
case 6:
return y, height - 1 - x
case 7:
return width - 1 - y, height - 1 - x
case 8:
return width - 1 - y, x
default:
return x, y
}
}
func flattenWhite(source image.Image) *image.NRGBA {
bounds := source.Bounds()
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
for y := 0; y < bounds.Dy(); y++ {
for x := 0; x < bounds.Dx(); x++ {
r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA()
output.SetNRGBA(x, y, color.NRGBA{
R: uint8((r + 0xffff - a) / 0x101),
G: uint8((g + 0xffff - a) / 0x101),
B: uint8((b + 0xffff - a) / 0x101),
A: 0xff,
})
}
}
return output
}
@@ -0,0 +1,237 @@
package transform
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
"image/jpeg"
_ "image/png"
"math"
"net/http"
"github.com/deepteams/webp"
)
const (
// DefaultJPEGQuality is used for thumbnail and crop variants.
DefaultJPEGQuality = 85
// DefaultWebPQuality keeps the optimized variant visually close to the source while reducing its size.
DefaultWebPQuality = 82
// DefaultWebPMethod selects the encoder's balanced quality and CPU mode.
DefaultWebPMethod = 4
)
var (
ErrInvalidConfig = errors.New("invalid image transform configuration")
ErrUnsupportedMedia = errors.New("unsupported image media type")
ErrPayloadTooLarge = errors.New("image exceeds configured limits")
ErrInvalidImage = errors.New("invalid image data")
)
type Config struct {
MaxBytes int64
MaxPixels int64
JPEGQuality int
WebPQuality float32
WebPMethod int
}
type Variant struct {
Bytes []byte
ContentType string
Width int
Height int
SHA256 string
}
type Result struct {
Original Variant
Thumbnail Variant
Crop Variant
Optimized Variant
Orientation int
}
type Processor struct {
config Config
}
func DefaultConfig(maxBytes, maxPixels int64) Config {
return Config{
MaxBytes: maxBytes, MaxPixels: maxPixels,
JPEGQuality: DefaultJPEGQuality, WebPQuality: DefaultWebPQuality, WebPMethod: DefaultWebPMethod,
}
}
func NewProcessor(config Config) (*Processor, error) {
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.JPEGQuality < 1 || config.JPEGQuality > 100 ||
config.WebPQuality < 1 || config.WebPQuality > 100 || config.WebPMethod < 0 || config.WebPMethod > 6 {
return nil, ErrInvalidConfig
}
return &Processor{config: config}, nil
}
func (p *Processor) Process(ctx context.Context, data []byte, declaredContentType string) (Result, error) {
if err := contextError(ctx); err != nil {
return Result{}, err
}
img, format, width, height, orientation, err := p.decode(data, declaredContentType)
if err != nil {
return Result{}, err
}
if err := contextError(ctx); err != nil {
return Result{}, err
}
oriented := applyOrientation(img, orientation)
if err := contextError(ctx); err != nil {
return Result{}, err
}
opaque := flattenWhite(oriented)
if err := contextError(ctx); err != nil {
return Result{}, err
}
result := Result{
Original: variant(data, format, width, height),
Orientation: orientation,
}
thumbnail := fit(opaque, 256, 256, false)
result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality)
if err != nil {
return Result{}, err
}
if err := contextError(ctx); err != nil {
return Result{}, err
}
crop := centeredSquare(opaque)
crop = resize(crop, 128, 128)
result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality)
if err != nil {
return Result{}, err
}
if err := contextError(ctx); err != nil {
return Result{}, err
}
optimized := fit(opaque, 1280, 1280, false)
result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod)
if err != nil {
return Result{}, err
}
if err := contextError(ctx); err != nil {
return Result{}, err
}
return result, nil
}
func (p *Processor) decode(data []byte, declaredContentType string) (image.Image, string, int, int, int, error) {
if declaredContentType != "image/jpeg" && declaredContentType != "image/png" {
return nil, "", 0, 0, 0, ErrUnsupportedMedia
}
if len(data) == 0 {
return nil, "", 0, 0, 0, ErrInvalidImage
}
if int64(len(data)) > p.config.MaxBytes {
return nil, "", 0, 0, 0, ErrPayloadTooLarge
}
if http.DetectContentType(data) != declaredContentType {
return nil, "", 0, 0, 0, ErrUnsupportedMedia
}
config, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 {
return nil, "", 0, 0, 0, ErrInvalidImage
}
if int64(config.Width) > p.config.MaxPixels/int64(config.Height) {
return nil, "", 0, 0, 0, ErrPayloadTooLarge
}
img, decodedFormat, err := image.Decode(bytes.NewReader(data))
if err != nil || decodedFormat != format {
return nil, "", 0, 0, 0, ErrInvalidImage
}
orientation := 1
if format == "jpeg" {
orientation, err = jpegOrientation(data)
if err != nil {
return nil, "", 0, 0, 0, ErrInvalidImage
}
}
return img, declaredContentType, config.Width, config.Height, orientation, nil
}
func contextError(ctx context.Context) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
return nil
}
}
func contentType(format string) string {
switch format {
case "jpeg":
return "image/jpeg"
case "png":
return "image/png"
default:
return ""
}
}
func encodeJPEG(img image.Image, quality int) (Variant, error) {
var output bytes.Buffer
if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: quality}); err != nil {
return Variant{}, err
}
bounds := img.Bounds()
return variant(output.Bytes(), "image/jpeg", bounds.Dx(), bounds.Dy()), nil
}
func encodeWebP(img image.Image, quality float32, method int) (Variant, error) {
var output bytes.Buffer
options := &webp.EncoderOptions{Quality: quality, Method: method}
if err := webp.Encode(&output, img, options); err != nil {
return Variant{}, err
}
bounds := img.Bounds()
return variant(output.Bytes(), "image/webp", bounds.Dx(), bounds.Dy()), nil
}
func variant(data []byte, contentType string, width, height int) Variant {
digest := sha256.Sum256(data)
return Variant{
Bytes: data, ContentType: contentType, Width: width, Height: height,
SHA256: hex.EncodeToString(digest[:]),
}
}
// EstimateMemoryWeight returns a conservative reservation for the input, decoded source,
// orientation and alpha copies, variant working buffer, and encoded variant buffers.
func EstimateMemoryWeight(inputBytes int64, width, height int) (int64, error) {
if inputBytes < 0 || width < 1 || height < 1 || int64(width) > math.MaxInt64/int64(height) {
return 0, ErrInvalidImage
}
pixels := int64(width) * int64(height)
optimizedPixels := fittedPixels(width, height, 1280, 1280)
thumbnailPixels := fittedPixels(width, height, 256, 256)
const cropPixels int64 = 128 * 128
if pixels > (math.MaxInt64-inputBytes)/16 {
return 0, ErrPayloadTooLarge
}
weight := inputBytes + pixels*16
variants := (optimizedPixels + thumbnailPixels + cropPixels) * 4
if variants > math.MaxInt64-weight {
return 0, ErrPayloadTooLarge
}
return weight + variants, nil
}
func fittedPixels(width, height, maxWidth, maxHeight int) int64 {
newWidth, newHeight := fitDimensions(width, height, maxWidth, maxHeight, false)
return int64(newWidth) * int64(newHeight)
}
@@ -0,0 +1,236 @@
package transform
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"image"
"image/color"
"image/jpeg"
"image/png"
"testing"
"github.com/deepteams/webp"
)
func TestProcessProducesExpectedVariants(t *testing.T) {
source := image.NewNRGBA(image.Rect(0, 0, 320, 200))
fillGradient(source)
data := encodePNG(t, source)
processor := newTestProcessor(t, int64(len(data))+1, 320*200)
result, err := processor.Process(context.Background(), data, "image/png")
if err != nil {
t.Fatalf("process image: %v", err)
}
assertVariant(t, result.Original, "image/png", 320, 200)
assertVariant(t, result.Thumbnail, "image/jpeg", 256, 160)
assertVariant(t, result.Crop, "image/jpeg", 128, 128)
assertVariant(t, result.Optimized, "image/webp", 320, 200)
if !bytes.Equal(result.Original.Bytes, data) {
t.Fatal("original bytes changed")
}
digest := sha256.Sum256(data)
if result.Original.SHA256 != hex.EncodeToString(digest[:]) {
t.Fatalf("unexpected original digest: %s", result.Original.SHA256)
}
if _, err := webp.Decode(bytes.NewReader(result.Optimized.Bytes)); err != nil {
t.Fatalf("decode optimized WebP: %v", err)
}
}
func TestProcessDoesNotUpscaleFitVariants(t *testing.T) {
source := image.NewNRGBA(image.Rect(0, 0, 64, 32))
fillGradient(source)
data := encodePNG(t, source)
processor := newTestProcessor(t, int64(len(data))+1, 64*32)
result, err := processor.Process(context.Background(), data, "image/png")
if err != nil {
t.Fatalf("process image: %v", err)
}
assertVariant(t, result.Thumbnail, "image/jpeg", 64, 32)
assertVariant(t, result.Optimized, "image/webp", 64, 32)
assertVariant(t, result.Crop, "image/jpeg", 128, 128)
}
func TestFlattenWhiteCompositesAlpha(t *testing.T) {
source := image.NewNRGBA(image.Rect(0, 0, 2, 1))
source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0})
source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128})
result := flattenWhite(source)
if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) {
t.Fatalf("transparent pixel: got %#v", got)
}
if got := result.NRGBAAt(1, 0); got != (color.NRGBA{R: 177, G: 152, B: 127, A: 255}) {
t.Fatalf("semi-transparent pixel: got %#v", got)
}
}
func TestProcessAppliesEXIFOrientation(t *testing.T) {
source := image.NewNRGBA(image.Rect(0, 0, 3, 2))
fillGradient(source)
data := injectOrientation(t, encodeJPEGForTest(t, source), 6)
processor := newTestProcessor(t, int64(len(data))+1, 3*2)
result, err := processor.Process(context.Background(), data, "image/jpeg")
if err != nil {
t.Fatalf("process image: %v", err)
}
if result.Orientation != 6 {
t.Fatalf("orientation: got %d, want 6", result.Orientation)
}
assertVariant(t, result.Original, "image/jpeg", 3, 2)
assertVariant(t, result.Thumbnail, "image/jpeg", 2, 3)
assertVariant(t, result.Optimized, "image/webp", 2, 3)
}
func TestProcessRejectsInvalidInputs(t *testing.T) {
valid := encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 11, 10)))
tests := []struct {
name string
data []byte
contentType string
maxPixels int64
want error
}{
{name: "MIME mismatch", data: valid, contentType: "image/jpeg", maxPixels: 200, want: ErrUnsupportedMedia},
{name: "unsupported MIME", data: valid, contentType: "image/gif", maxPixels: 200, want: ErrUnsupportedMedia},
{name: "pixel limit", data: valid, contentType: "image/png", maxPixels: 100, want: ErrPayloadTooLarge},
{name: "corrupted", data: valid[:len(valid)-12], contentType: "image/png", maxPixels: 200, want: ErrInvalidImage},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
processor := newTestProcessor(t, int64(len(valid))+1, test.maxPixels)
_, err := processor.Process(context.Background(), test.data, test.contentType)
if !errors.Is(err, test.want) {
t.Fatalf("error: got %v, want %v", err, test.want)
}
})
}
}
func TestProcessIsDeterministic(t *testing.T) {
source := image.NewNRGBA(image.Rect(0, 0, 48, 31))
fillGradient(source)
data := encodePNG(t, source)
processor := newTestProcessor(t, int64(len(data))+1, 48*31)
first, err := processor.Process(context.Background(), data, "image/png")
if err != nil {
t.Fatalf("first process: %v", err)
}
second, err := processor.Process(context.Background(), data, "image/png")
if err != nil {
t.Fatalf("second process: %v", err)
}
for name, variants := range map[string][2]Variant{
"thumbnail": {first.Thumbnail, second.Thumbnail},
"crop": {first.Crop, second.Crop},
"optimized": {first.Optimized, second.Optimized},
} {
if !bytes.Equal(variants[0].Bytes, variants[1].Bytes) {
t.Fatalf("%s bytes differ", name)
}
}
}
func TestProcessHonorsCanceledContext(t *testing.T) {
processor := newTestProcessor(t, 1024, 100)
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := processor.Process(ctx, []byte("unused"), "image/png")
if !errors.Is(err, context.Canceled) {
t.Fatalf("error: got %v, want %v", err, context.Canceled)
}
}
func TestEstimateMemoryWeight(t *testing.T) {
weight, err := EstimateMemoryWeight(1000, 320, 200)
if err != nil {
t.Fatalf("estimate memory: %v", err)
}
want := int64(1000 + 320*200*16 + (320*200+256*160+128*128)*4)
if weight != want {
t.Fatalf("weight: got %d, want %d", weight, want)
}
}
func newTestProcessor(t *testing.T, maxBytes, maxPixels int64) *Processor {
t.Helper()
processor, err := NewProcessor(DefaultConfig(maxBytes, maxPixels))
if err != nil {
t.Fatalf("new processor: %v", err)
}
return processor
}
func assertVariant(t *testing.T, variant Variant, contentType string, width, height int) {
t.Helper()
if variant.ContentType != contentType || variant.Width != width || variant.Height != height {
t.Fatalf("variant: got type=%s dimensions=%dx%d, want type=%s dimensions=%dx%d",
variant.ContentType, variant.Width, variant.Height, contentType, width, height)
}
if len(variant.Bytes) == 0 || variant.SHA256 == "" {
t.Fatal("variant has no bytes or digest")
}
}
func fillGradient(img *image.NRGBA) {
for y := 0; y < img.Bounds().Dy(); y++ {
for x := 0; x < img.Bounds().Dx(); x++ {
img.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
}
}
}
func encodePNG(t *testing.T, img image.Image) []byte {
t.Helper()
var output bytes.Buffer
if err := png.Encode(&output, img); err != nil {
t.Fatalf("encode PNG: %v", err)
}
return output.Bytes()
}
func encodeJPEGForTest(t *testing.T, img image.Image) []byte {
t.Helper()
var output bytes.Buffer
if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: 100}); err != nil {
t.Fatalf("encode JPEG: %v", err)
}
return output.Bytes()
}
func injectOrientation(t *testing.T, data []byte, orientation uint16) []byte {
t.Helper()
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
t.Fatal("not a JPEG")
}
tiff := make([]byte, 26)
copy(tiff[:2], "II")
binary.LittleEndian.PutUint16(tiff[2:4], 42)
binary.LittleEndian.PutUint32(tiff[4:8], 8)
binary.LittleEndian.PutUint16(tiff[8:10], 1)
binary.LittleEndian.PutUint16(tiff[10:12], 0x0112)
binary.LittleEndian.PutUint16(tiff[12:14], 3)
binary.LittleEndian.PutUint32(tiff[14:18], 1)
binary.LittleEndian.PutUint16(tiff[18:20], orientation)
payload := append([]byte("Exif\x00\x00"), tiff...)
segment := []byte{0xff, 0xe1, 0, 0}
binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2))
result := make([]byte, 0, len(data)+len(segment)+len(payload))
result = append(result, data[:2]...)
result = append(result, segment...)
result = append(result, payload...)
return append(result, data[2:]...)
}
@@ -0,0 +1,105 @@
package transform
import (
"image"
"image/color"
)
func fit(source image.Image, maxWidth, maxHeight int, upscale bool) *image.NRGBA {
bounds := source.Bounds()
width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale)
return resize(source, width, height)
}
func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) {
if !upscale && width <= maxWidth && height <= maxHeight {
return width, height
}
if int64(width)*int64(maxHeight) > int64(height)*int64(maxWidth) {
newHeight := max(1, int(int64(height)*int64(maxWidth)/int64(width)))
return maxWidth, newHeight
}
newWidth := max(1, int(int64(width)*int64(maxHeight)/int64(height)))
return newWidth, maxHeight
}
func centeredSquare(source image.Image) image.Image {
bounds := source.Bounds()
size := min(bounds.Dx(), bounds.Dy())
left := bounds.Min.X + (bounds.Dx()-size)/2
top := bounds.Min.Y + (bounds.Dy()-size)/2
return croppedImage{source: source, bounds: image.Rect(0, 0, size, size), offset: image.Pt(left, top)}
}
type croppedImage struct {
source image.Image
bounds image.Rectangle
offset image.Point
}
func (i croppedImage) ColorModel() color.Model { return i.source.ColorModel() }
func (i croppedImage) Bounds() image.Rectangle { return i.bounds }
func (i croppedImage) At(x, y int) color.Color {
return i.source.At(i.offset.X+x, i.offset.Y+y)
}
func resize(source image.Image, width, height int) *image.NRGBA {
bounds := source.Bounds()
if bounds.Dx() == width && bounds.Dy() == height {
return copyNRGBA(source)
}
output := image.NewNRGBA(image.Rect(0, 0, width, height))
for y := 0; y < height; y++ {
y0, y1, fy := sampleCoordinate(y, height, bounds.Dy())
for x := 0; x < width; x++ {
x0, x1, fx := sampleCoordinate(x, width, bounds.Dx())
c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA)
c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA)
c01 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y1)).(color.NRGBA)
c11 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y1)).(color.NRGBA)
output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy))
}
}
return output
}
func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) {
coordinate := ((int64(2*destination+1)*int64(sourceSize))<<15)/int64(destinationSize) - 1<<15
if coordinate <= 0 {
return 0, min(1, sourceSize-1), 0
}
maximum := int64(sourceSize-1) << 16
if coordinate >= maximum {
return sourceSize - 1, sourceSize - 1, 0
}
first := int(coordinate >> 16)
return first, first + 1, uint32(coordinate & 0xffff)
}
func interpolate(c00, c10, c01, c11 color.NRGBA, fx, fy uint32) color.NRGBA {
return color.NRGBA{
R: interpolateChannel(c00.R, c10.R, c01.R, c11.R, fx, fy),
G: interpolateChannel(c00.G, c10.G, c01.G, c11.G, fx, fy),
B: interpolateChannel(c00.B, c10.B, c01.B, c11.B, fx, fy),
A: interpolateChannel(c00.A, c10.A, c01.A, c11.A, fx, fy),
}
}
func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 {
const one uint64 = 1 << 16
top := uint64(c00)*(one-uint64(fx)) + uint64(c10)*uint64(fx)
bottom := uint64(c01)*(one-uint64(fx)) + uint64(c11)*uint64(fx)
value := top*(one-uint64(fy)) + bottom*uint64(fy)
return uint8((value + 1<<31) >> 32)
}
func copyNRGBA(source image.Image) *image.NRGBA {
bounds := source.Bounds()
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
for y := 0; y < bounds.Dy(); y++ {
for x := 0; x < bounds.Dx(); x++ {
output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y))
}
}
return output
}
@@ -0,0 +1,452 @@
package worker
import (
"bytes"
"context"
"errors"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"log/slog"
"math"
"path"
"strings"
"sync"
"time"
"golang.org/x/sync/semaphore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
)
type Repository interface {
AttachmentJob(context.Context, string) (model.AttachmentJob, error)
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
}
type Broker interface {
Read(context.Context) ([]redisstream.Delivery, error)
Reclaim(context.Context, string) ([]redisstream.Delivery, string, error)
}
type ObjectStore interface {
Get(context.Context, string) (io.ReadCloser, error)
Put(context.Context, string, string, []byte) error
Delete(context.Context, string) error
}
type ImageProcessor interface {
Process(context.Context, []byte, string) (transform.Result, error)
}
type Metrics interface {
ObserveStarted()
ObserveFinished(string, time.Duration)
ObserveRetry()
ObserveReclaimed(int)
}
type Config struct {
WorkerID string
Concurrency int
LeaseDuration time.Duration
ProcessingTimeout time.Duration
ReclaimInterval time.Duration
MaxAttempts int
RetryBase time.Duration
RetryMax time.Duration
MaxBytes int64
MaxPixels int64
MemoryBytes int64
}
type FaultPoint string
const (
FaultNone FaultPoint = ""
FaultBeforeSideEffect FaultPoint = "before_side_effect"
FaultAfterSideEffect FaultPoint = "after_side_effect"
FaultAfterCommit FaultPoint = "after_commit"
)
var ErrInjectedFailure = errors.New("injected worker failure")
type Worker struct {
logger *slog.Logger
repository Repository
broker Broker
objects ObjectStore
processor ImageProcessor
metrics Metrics
config Config
memory *semaphore.Weighted
now func() time.Time
}
func New(
logger *slog.Logger,
repository Repository,
broker Broker,
objects ObjectStore,
processor ImageProcessor,
metrics Metrics,
config Config,
) (*Worker, error) {
if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil {
return nil, errors.New("worker dependencies are required")
}
if err := validateConfig(config); err != nil {
return nil, err
}
if metrics == nil {
metrics = nopMetrics{}
}
return &Worker{
logger: logger, repository: repository, broker: broker, objects: objects,
processor: processor, metrics: metrics, config: config,
memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now,
}, nil
}
func validateConfig(config Config) error {
if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 ||
config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration ||
config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 ||
config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 {
return errors.New("invalid worker configuration")
}
const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128
if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 ||
config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes {
return errors.New("worker memory must fit one maximum image")
}
return nil
}
func (w *Worker) Run(ctx context.Context) error {
lastReclaim := time.Time{}
reclaimCursor := "0-0"
for ctx.Err() == nil {
deliveries, err := w.broker.Read(ctx)
if err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
w.processBatch(ctx, deliveries)
if w.now().Sub(lastReclaim) < w.config.ReclaimInterval {
continue
}
recovery, err := w.repository.RecoverExpiredAttachmentJobs(
ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase),
)
if err != nil {
return fmt.Errorf("recover expired attachment jobs: %w", err)
}
if recovery.Requeued > 0 || recovery.Failed > 0 {
w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed)
}
deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor)
if err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
w.metrics.ObserveReclaimed(len(deliveries))
w.processBatch(ctx, deliveries)
reclaimCursor = next
if reclaimCursor == "" {
reclaimCursor = "0-0"
}
lastReclaim = w.now()
}
return nil
}
func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) {
limit := make(chan struct{}, w.config.Concurrency)
var group sync.WaitGroup
for _, delivery := range deliveries {
if ctx.Err() != nil {
break
}
limit <- struct{}{}
group.Add(1)
go func(delivery redisstream.Delivery) {
defer group.Done()
defer func() { <-limit }()
if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil {
w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err)
}
}(delivery)
}
group.Wait()
}
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error {
started := w.now()
w.metrics.ObserveStarted()
result := "transient_error"
defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }()
if delivery.DecodeError != nil {
result = "invalid_job"
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
}
envelope := delivery.Envelope
lease, err := w.repository.ClaimAttachmentJob(
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
w.config.WorkerID, w.config.LeaseDuration,
)
if err != nil {
result, err = w.resolveClaim(ctx, delivery, envelope, err)
return err
}
if fault == FaultBeforeSideEffect {
result = "injected_failure"
return ErrInjectedFailure
}
processingCtx, cancel := context.WithTimeout(ctx, w.config.ProcessingTimeout)
variants, err := w.process(processingCtx, lease)
cancel()
if err != nil {
result, err = w.resolveProcessingFailure(ctx, delivery, lease, err)
return err
}
if fault == FaultAfterSideEffect {
result = "injected_failure"
return ErrInjectedFailure
}
mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants)
if err != nil {
return fmt.Errorf("complete attachment job: %w", err)
}
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
}
}
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
return fmt.Errorf("unexpected completion result %q", mutation)
}
if fault == FaultAfterCommit {
result = "injected_failure"
return ErrInjectedFailure
}
if err := delivery.Ack(ctx); err != nil {
return err
}
result = "ready"
return nil
}
func (w *Worker) resolveClaim(
ctx context.Context,
delivery redisstream.Delivery,
envelope redisstream.Envelope,
claimErr error,
) (string, error) {
switch {
case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput):
return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline):
return "duplicate", delivery.Ack(ctx)
case errors.Is(claimErr, model.ErrJobBusy):
return "duplicate", delivery.Ack(ctx)
case errors.Is(claimErr, model.ErrJobNotDue):
return "not_due", nil
case errors.Is(claimErr, model.ErrConflict):
job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String())
if err != nil {
return "transient_error", fmt.Errorf("inspect conflicting job: %w", err)
}
switch job.Status {
case model.AttachmentJobReady:
return "duplicate", delivery.Ack(ctx)
case model.AttachmentJobFailed:
code := "processing_failed"
if job.ErrorCode != nil {
code = *job.ErrorCode
}
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
default:
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
}
default:
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
}
}
func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) {
body, err := w.objects.Get(ctx, lease.Attachment.StagingKey)
if err != nil {
return nil, err
}
defer func() { _ = body.Close() }()
data, err := io.ReadAll(io.LimitReader(body, w.config.MaxBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > w.config.MaxBytes {
return nil, transform.ErrPayloadTooLarge
}
imageConfig, _, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil {
return nil, transform.ErrInvalidImage
}
weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height)
if err != nil {
return nil, err
}
if weight > w.config.MemoryBytes {
return nil, errWorkerCapacity
}
if err := w.memory.Acquire(ctx, weight); err != nil {
return nil, err
}
defer w.memory.Release(weight)
processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType)
if err != nil {
return nil, err
}
variants := []struct {
name string
value transform.Variant
}{
{name: "original", value: processed.Original},
{name: "thumbnail", value: processed.Thumbnail},
{name: "crop", value: processed.Crop},
{name: "optimized", value: processed.Optimized},
}
result := make([]model.Variant, 0, len(variants))
for _, item := range variants {
objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value)
if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil {
return nil, err
}
result = append(result, model.Variant{
Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)),
SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height,
ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion,
})
}
return result, nil
}
var errWorkerCapacity = errors.New("image exceeds worker memory budget")
func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string {
extension := ".bin"
switch value.ContentType {
case "image/png":
extension = ".png"
case "image/jpeg":
extension = ".jpg"
case "image/webp":
extension = ".webp"
}
return path.Join(
"attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion),
name, value.SHA256+extension,
)
}
func (w *Worker) resolveProcessingFailure(
ctx context.Context,
delivery redisstream.Delivery,
lease model.AttachmentJobLease,
processingErr error,
) (string, error) {
code, permanent := processingError(processingErr)
if permanent || lease.Job.Attempt >= w.config.MaxAttempts {
if !permanent {
code = "retry_exhausted"
}
mutation, err := w.repository.FailAttachmentJob(ctx, lease, code)
if err != nil {
return "transient_error", fmt.Errorf("fail attachment job: %w", err)
}
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
return "duplicate", delivery.Ack(ctx)
}
if mutation != model.JobMutationApplied {
return "transient_error", fmt.Errorf("unexpected failure result %q", mutation)
}
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
}
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt)
if err != nil {
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
}
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
return "duplicate", delivery.Ack(ctx)
}
if mutation != model.JobMutationApplied {
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
}
w.metrics.ObserveRetry()
return "retry", delivery.Ack(ctx)
}
func processingError(err error) (string, bool) {
switch {
case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia):
return "invalid_image", true
case errors.Is(err, transform.ErrPayloadTooLarge):
return "image_too_large", true
case errors.Is(err, objectstore.ErrNotFound):
return "source_missing", true
case errors.Is(err, errWorkerCapacity):
return "worker_capacity", true
default:
return "processing_failed", false
}
}
func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
delay := w.config.RetryBase
for step := 1; step < attempt && delay < w.config.RetryMax; step++ {
if delay > w.config.RetryMax/2 {
delay = w.config.RetryMax
break
}
delay *= 2
}
if delay > w.config.RetryMax {
delay = w.config.RetryMax
}
var hash uint64
for index := range jobID {
hash = hash*131 + uint64(jobID[index])
}
jitterRange := min(delay/4, w.config.RetryMax-delay)
if jitterRange <= 0 {
return delay
}
jitter := time.Duration(hash % uint64(jitterRange))
if delay > time.Duration(math.MaxInt64)-jitter {
return delay
}
return delay + jitter
}
type nopMetrics struct{}
func (nopMetrics) ObserveStarted() {}
func (nopMetrics) ObserveFinished(string, time.Duration) {}
func (nopMetrics) ObserveRetry() {}
func (nopMetrics) ObserveReclaimed(int) {}
@@ -0,0 +1,56 @@
package worker
import (
"errors"
"testing"
"time"
"github.com/stretchr/testify/require"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
)
func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) {
config := validConfig()
require.NoError(t, validateConfig(config))
config.MemoryBytes = 384 * 1024 * 1024
require.ErrorContains(t, validateConfig(config), "memory must fit")
}
func TestRetryDelayIsBounded(t *testing.T) {
worker := &Worker{config: Config{RetryBase: time.Second, RetryMax: 4 * time.Second}}
for attempt := 1; attempt <= 10; attempt++ {
delay := worker.retryDelay("11111111-1111-4111-8111-111111111111", attempt)
require.GreaterOrEqual(t, delay, time.Second)
require.LessOrEqual(t, delay, 4*time.Second)
}
}
func TestProcessingErrorClassification(t *testing.T) {
tests := []struct {
err error
code string
permanent bool
}{
{err: transform.ErrInvalidImage, code: "invalid_image", permanent: true},
{err: transform.ErrPayloadTooLarge, code: "image_too_large", permanent: true},
{err: objectstore.ErrNotFound, code: "source_missing", permanent: true},
{err: errors.New("temporary S3 failure"), code: "processing_failed", permanent: false},
}
for _, test := range tests {
code, permanent := processingError(test.err)
require.Equal(t, test.code, code)
require.Equal(t, test.permanent, permanent)
}
}
func validConfig() Config {
return Config{
WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second,
ProcessingTimeout: 20 * time.Second, ReclaimInterval: 5 * time.Second,
MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second,
MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024,
}
}
+37 -8
View File
@@ -6,6 +6,7 @@ import (
"mime"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
@@ -46,6 +47,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
ChatID: chatID, OwnerID: user.ID, Filename: filename,
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")),
})
if err != nil {
if errors.Is(err, model.ErrCapacity) {
@@ -58,7 +60,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
}
result = "success"
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
writeJSON(w, http.StatusCreated, uploaded)
writeJSON(w, http.StatusAccepted, uploaded)
return nil
}
@@ -108,20 +110,21 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
return authErr
}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || !originalVariant(query) {
variant, valid := attachmentVariant(query)
if err != nil || !valid {
return invalidRequest()
}
attachmentID, apiErr := attachmentIDFromRequest(r)
if apiErr != nil {
return apiErr
}
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID)
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID, variant)
if err != nil {
return attachmentError(err)
}
defer func() { _ = content.Body.Close() }()
w.Header().Set("Content-Type", content.Attachment.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10))
w.Header().Set("Content-Type", content.Variant.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(content.Variant.SizeBytes, 10))
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, content.Body); err != nil {
@@ -130,11 +133,35 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
return nil
}
func originalVariant(query url.Values) bool {
func attachmentVariant(query url.Values) (string, bool) {
if len(query) == 0 {
return true
return "original", true
}
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
if len(query) != 1 || len(query["variant"]) != 1 {
return "", false
}
switch query["variant"][0] {
case "original", "thumbnail", "crop", "optimized":
return query["variant"][0], true
default:
return "", false
}
}
var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
func traceparent(value string) string {
if traceparentPattern.MatchString(value) {
return value
}
return ""
}
func tracestate(value string) string {
if len(value) <= 512 {
return value
}
return ""
}
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
@@ -158,6 +185,8 @@ func attachmentError(err error) *apiError {
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
case errors.Is(err, model.ErrCapacity):
return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
case errors.Is(err, model.ErrConflict):
return &apiError{http.StatusConflict, errorCodeConflict, "attachment is not ready"}
default:
return unavailable()
}
@@ -28,7 +28,7 @@ type MessageRepository interface {
type AttachmentService interface {
Upload(context.Context, attachment.Upload) (model.Attachment, error)
Metadata(context.Context, string, string) (model.Attachment, error)
Content(context.Context, string, string) (attachment.Content, error)
Content(context.Context, string, string, string) (attachment.Content, error)
}
type HealthChecker interface {
+6 -2
View File
@@ -87,8 +87,12 @@ func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model
func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) {
return s.uploaded, s.err
}
func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) {
return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err
func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) {
return attachment.Content{
Attachment: s.uploaded,
Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes},
Body: io.NopCloser(strings.NewReader("content")),
}, s.err
}
type sessionStub struct {
+6
View File
@@ -15,4 +15,10 @@ var (
ErrPayloadTooLarge = errors.New("payload too large")
ErrCapacity = errors.New("capacity exhausted")
ErrUnavailable = errors.New("dependency unavailable")
ErrStaleLease = errors.New("stale lease")
ErrStaleAttempt = errors.New("stale attempt")
ErrStalePipeline = errors.New("stale pipeline")
ErrJobBusy = errors.New("job is already leased")
ErrJobComplete = errors.New("job is already complete")
ErrJobNotDue = errors.New("job is not due")
)
+94 -24
View File
@@ -6,9 +6,21 @@ import (
)
const (
AttachmentPending = "pending"
AttachmentReady = "ready"
AttachmentDeleting = "deleting"
AttachmentPending = "pending"
AttachmentQueued = "queued"
AttachmentProcessing = "processing"
AttachmentReady = "ready"
AttachmentFailed = "failed"
AttachmentDeleting = "deleting"
AttachmentJobQueued = "queued"
AttachmentJobProcessing = "processing"
AttachmentJobReady = "ready"
AttachmentJobFailed = "failed"
JobMutationApplied JobMutationResult = "applied"
JobMutationAlreadyComplete JobMutationResult = "already_complete"
JobMutationStale JobMutationResult = "stale"
)
const timestampLayout = "2006-01-02T15:04:05.000000Z"
@@ -42,21 +54,23 @@ type Message struct {
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
ReadyAt *time.Time `json:"-"`
Referenced bool `json:"-"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
StagingKey string `json:"-"`
ReadyAt *time.Time `json:"-"`
Referenced bool `json:"-"`
PipelineVersion int `json:"-"`
}
type StoredObject struct {
@@ -66,12 +80,68 @@ type StoredObject struct {
}
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Width int `json:"width"`
Height int `json:"height"`
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Width int `json:"width"`
Height int `json:"height"`
ObjectKey string `json:"-"`
PipelineVersion int `json:"-"`
}
type AttachmentJob struct {
ID string
AttachmentID string
PipelineVersion int
Attempt int
Status string
AvailableAt time.Time
LeaseToken string
LeaseOwner string
LeaseUntil *time.Time
ErrorCode *string
CreatedAt time.Time
UpdatedAt time.Time
}
type AttachmentJobMessage struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
}
type AttachmentJobLease struct {
Job AttachmentJob
Attachment Attachment
}
type OutboxEvent struct {
ID string
Topic string
PartitionKey string
Payload json.RawMessage
JobID string
Attempt int
AvailableAt time.Time
PublishTries int
LeaseToken string
LeaseUntil *time.Time
LastErrorCode *string
CreatedAt time.Time
}
type JobMutationResult string
type RecoveryResult struct {
Requeued int
Failed int
}
type Position struct {
+174
View File
@@ -0,0 +1,174 @@
package outbox
import (
"context"
"errors"
"fmt"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
)
var errLeaseLost = errors.New("outbox lease lost")
type Store interface {
ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error)
MarkOutboxPublished(context.Context, string, string) (bool, error)
ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error)
RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error)
}
type Publisher interface {
PublishConfirmed(context.Context, []byte) (string, error)
}
type Metrics interface {
ObservePublished()
ObservePublishFailure()
ObserveRecovered(int)
}
type Config struct {
BatchSize int
LeaseDuration time.Duration
PollInterval time.Duration
RecoveryInterval time.Duration
RetryDelay time.Duration
}
type Dispatcher struct {
store Store
publisher Publisher
metrics Metrics
config Config
now func() time.Time
}
func New(store Store, publisher Publisher, metrics Metrics, config Config) (*Dispatcher, error) {
if store == nil {
return nil, errors.New("outbox store is required")
}
if publisher == nil {
return nil, errors.New("outbox publisher is required")
}
if err := config.validate(); err != nil {
return nil, err
}
if metrics == nil {
metrics = nopMetrics{}
}
return &Dispatcher{store: store, publisher: publisher, metrics: metrics, config: config, now: time.Now}, nil
}
func (c Config) validate() error {
switch {
case c.BatchSize <= 0:
return errors.New("outbox batch size must be positive")
case c.LeaseDuration <= 0:
return errors.New("outbox lease duration must be positive")
case c.PollInterval <= 0:
return errors.New("outbox poll interval must be positive")
case c.RecoveryInterval <= 0:
return errors.New("outbox recovery interval must be positive")
case c.RetryDelay <= 0:
return errors.New("outbox retry delay must be positive")
default:
return nil
}
}
func (d *Dispatcher) Run(ctx context.Context) error {
if ctx.Err() != nil {
return nil
}
pollTicker := time.NewTicker(d.config.PollInterval)
recoveryTicker := time.NewTicker(d.config.RecoveryInterval)
defer pollTicker.Stop()
defer recoveryTicker.Stop()
if err := d.RecoverExpired(ctx); err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
for {
if _, err := d.DispatchOnce(ctx); err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
select {
case <-ctx.Done():
return nil
case <-pollTicker.C:
case <-recoveryTicker.C:
if err := d.RecoverExpired(ctx); err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
}
}
}
func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration)
if err != nil {
return 0, fmt.Errorf("claim due outbox events: %w", err)
}
published := 0
for _, event := range events {
if err := ctx.Err(); err != nil {
return published, nil
}
_, err := d.publisher.PublishConfirmed(ctx, event.Payload)
if err != nil {
if ctx.Err() != nil {
return published, nil
}
d.metrics.ObservePublishFailure()
nextAttemptAt := d.now().Add(d.config.RetryDelay)
released, releaseErr := d.store.ReleaseOutboxEvent(
ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
)
if releaseErr != nil {
return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr)
}
if !released {
return published, fmt.Errorf("release outbox event after publish failure: %w", errLeaseLost)
}
continue
}
marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken)
if err != nil {
return published, fmt.Errorf("mark outbox event published: %w", err)
}
if !marked {
return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost)
}
d.metrics.ObservePublished()
published++
}
return published, nil
}
func (d *Dispatcher) RecoverExpired(ctx context.Context) error {
recovered, err := d.store.RecoverExpiredOutboxEvents(ctx, d.now())
if err != nil {
return fmt.Errorf("recover expired outbox leases: %w", err)
}
d.metrics.ObserveRecovered(recovered)
return nil
}
type nopMetrics struct{}
func (nopMetrics) ObservePublished() {}
func (nopMetrics) ObservePublishFailure() {}
func (nopMetrics) ObserveRecovered(int) {}
+181
View File
@@ -0,0 +1,181 @@
package outbox
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"github.com/stretchr/testify/require"
)
func TestDispatcherPublishesAndMarksEvent(t *testing.T) {
store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}}
publisher := &publisherStub{messageID: "1-0"}
dispatcher := newTestDispatcher(t, store, publisher)
published, err := dispatcher.DispatchOnce(context.Background())
require.NoError(t, err)
require.Equal(t, 1, published)
require.Equal(t, []string{"event-1:lease-1"}, store.marked)
}
func TestDispatcherRetriesSafelyAfterMarkFailure(t *testing.T) {
event := model.OutboxEvent{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}
store := &storeStub{events: []model.OutboxEvent{event}, markErr: errors.New("database unavailable")}
publisher := &publisherStub{messageID: "1-0"}
dispatcher := newTestDispatcher(t, store, publisher)
_, err := dispatcher.DispatchOnce(context.Background())
require.Error(t, err)
require.Equal(t, 1, publisher.calls)
store.markErr = nil
store.events[0].LeaseToken = "lease-2"
publisher.messageID = "2-0"
published, err := dispatcher.DispatchOnce(context.Background())
require.NoError(t, err)
require.Equal(t, 1, published)
require.Equal(t, 2, publisher.calls)
require.JSONEq(t, string(event.Payload), string(publisher.payloads[0]))
require.JSONEq(t, string(publisher.payloads[0]), string(publisher.payloads[1]))
require.Equal(t, []string{"event-1:lease-2"}, store.marked)
}
func TestDispatcherReleasesPublishFailure(t *testing.T) {
store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}}
publisher := &publisherStub{err: errors.New("redis unavailable")}
dispatcher := newTestDispatcher(t, store, publisher)
dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) }
published, err := dispatcher.DispatchOnce(context.Background())
require.NoError(t, err)
require.Zero(t, published)
require.Equal(t, "event-1:lease-1:broker_publish_failed", store.released)
require.Equal(t, dispatcher.now().Add(time.Second), store.nextAttemptAt)
}
func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) {
store := &storeStub{
events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "stale-lease", Payload: testPayload()}},
markLost: true,
}
dispatcher := newTestDispatcher(t, store, &publisherStub{messageID: "1-0"})
published, err := dispatcher.DispatchOnce(context.Background())
require.ErrorIs(t, err, errLeaseLost)
require.Zero(t, published)
}
func TestDispatcherStopsAfterCancellation(t *testing.T) {
store := &storeStub{}
publisher := &publisherStub{}
dispatcher := newTestDispatcher(t, store, publisher)
ctx, cancel := context.WithCancel(context.Background())
cancel()
done := make(chan error, 1)
go func() { done <- dispatcher.Run(ctx) }()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(time.Second):
t.Fatal("dispatcher did not stop after cancellation")
}
}
func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatcher {
t.Helper()
dispatcher, err := New(store, publisher, nil, Config{
BatchSize: 10,
LeaseDuration: time.Minute,
PollInterval: time.Millisecond,
RecoveryInterval: time.Second,
RetryDelay: time.Second,
})
require.NoError(t, err)
return dispatcher
}
func testPayload() json.RawMessage {
return json.RawMessage(`{
"schema_version": 1,
"job_id": "11111111-1111-4111-8111-111111111111",
"type": "attachment.process.v1",
"attachment_id": "22222222-2222-4222-8222-222222222222",
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-10-22T12:00:00Z"
}`)
}
type storeStub struct {
mu sync.Mutex
events []model.OutboxEvent
markErr error
markLost bool
marked []string
released string
nextAttemptAt time.Time
recoveredCalls int
}
func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) {
s.mu.Lock()
defer s.mu.Unlock()
return append([]model.OutboxEvent(nil), s.events...), nil
}
func (s *storeStub) MarkOutboxPublished(_ context.Context, id, leaseToken string) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.markErr != nil {
return false, s.markErr
}
if s.markLost {
return false, nil
}
s.marked = append(s.marked, id+":"+leaseToken)
return true, nil
}
func (s *storeStub) ReleaseOutboxEvent(
_ context.Context,
id string,
leaseToken string,
code string,
nextAttemptAt time.Time,
) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.released = id + ":" + leaseToken + ":" + code
s.nextAttemptAt = nextAttemptAt
return true, nil
}
func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.recoveredCalls++
return 0, nil
}
type publisherStub struct {
mu sync.Mutex
messageID string
err error
calls int
payloads [][]byte
}
func (p *publisherStub) PublishConfirmed(_ context.Context, payload []byte) (string, error) {
p.mu.Lock()
defer p.mu.Unlock()
p.calls++
p.payloads = append(p.payloads, append([]byte(nil), payload...))
return p.messageID, p.err
}
+560
View File
@@ -0,0 +1,560 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
)
const attachmentJobTopic = "attachment.jobs"
const attachmentJobColumns = `
id, attachment_id, pipeline_version, attempt, status, available_at,
lease_token, lease_owner, lease_until, error_code, created_at, updated_at`
func (s *Store) QueueAttachmentWithJobAndOutbox(
ctx context.Context,
attachmentID string,
stagingKey string,
job model.AttachmentJob,
traceparent string,
tracestate string,
) (model.Attachment, error) {
if job.ID == "" || job.PipelineVersion < 1 {
return model.Attachment{}, model.ErrInvalidInput
}
if job.Attempt == 0 {
job.Attempt = 1
}
if job.Attempt != 1 {
return model.Attachment{}, model.ErrInvalidInput
}
now := time.Now().UTC()
if job.CreatedAt.IsZero() {
job.CreatedAt = now
}
if job.AvailableAt.IsZero() {
job.AvailableAt = now
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.Attachment{}, fmt.Errorf("begin queue attachment transaction: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
query := `
UPDATE attachments
SET status = 'queued', pipeline_version = $3, staging_key = $2,
ready_at = NULL, error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND object_key = $2 AND status = 'pending'
RETURNING ` + attachmentColumns
var attachment model.Attachment
err = scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.PipelineVersion), &attachment)
if errors.Is(err, pgx.ErrNoRows) {
return s.alreadyQueuedAttachment(ctx, tx, attachmentID, stagingKey, job)
}
if err != nil {
return model.Attachment{}, fmt.Errorf("queue attachment: %w", err)
}
job.AttachmentID = attachmentID
job.Status = model.AttachmentJobQueued
job.UpdatedAt = job.CreatedAt
if err := insertAttachmentJob(ctx, tx, job); err != nil {
return model.Attachment{}, err
}
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
return model.Attachment{}, err
}
if err := tx.Commit(ctx); err != nil {
return model.Attachment{}, fmt.Errorf("commit queued attachment: %w", err)
}
return publicAttachment(attachment), nil
}
func (s *Store) QueueAttachmentPipelineVersion(
ctx context.Context,
attachmentID string,
job model.AttachmentJob,
traceparent string,
tracestate string,
) (model.Attachment, error) {
if job.ID == "" || job.PipelineVersion < 2 {
return model.Attachment{}, model.ErrInvalidInput
}
job.Attempt = 1
now := time.Now().UTC()
job.CreatedAt = now
job.AvailableAt = now
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.Attachment{}, fmt.Errorf("begin queue attachment pipeline version: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
const selectAttachment = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 FOR UPDATE`
var attachment model.Attachment
if err := scanAttachment(tx.QueryRow(ctx, selectAttachment, attachmentID), &attachment); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrNotFound
}
return model.Attachment{}, fmt.Errorf("lock attachment for pipeline update: %w", err)
}
if attachment.Status != model.AttachmentReady || job.PipelineVersion != attachment.PipelineVersion+1 {
return model.Attachment{}, ErrConflict
}
job.AttachmentID = attachment.ID
job.Status = model.AttachmentJobQueued
job.UpdatedAt = now
const queueAttachment = `
UPDATE attachments
SET status = 'queued', pipeline_version = $2, staging_key = object_key,
ready_at = NULL, error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1
RETURNING ` + attachmentColumns
if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil {
return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err)
}
if err := insertAttachmentJob(ctx, tx, job); err != nil {
return model.Attachment{}, err
}
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
return model.Attachment{}, err
}
if err := tx.Commit(ctx); err != nil {
return model.Attachment{}, fmt.Errorf("commit queued attachment pipeline version: %w", err)
}
return publicAttachment(attachment), nil
}
func (s *Store) alreadyQueuedAttachment(
ctx context.Context,
tx pgx.Tx,
attachmentID string,
stagingKey string,
job model.AttachmentJob,
) (model.Attachment, error) {
query := `
SELECT ` + attachmentColumnsWithAlias("a") + `
FROM attachments a
JOIN attachment_jobs j
ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
WHERE a.id = $1 AND a.staging_key = $2 AND a.status IN ('queued', 'processing', 'ready')
AND j.id = $3 AND j.pipeline_version = $4`
var attachment model.Attachment
if err := scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.ID, job.PipelineVersion), &attachment); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, ErrConflict
}
return model.Attachment{}, fmt.Errorf("find queued attachment: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return model.Attachment{}, fmt.Errorf("commit repeated queue attachment: %w", err)
}
return publicAttachment(attachment), nil
}
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error {
const query = `
INSERT INTO attachment_jobs (
id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at
)
VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)`
if _, err := tx.Exec(
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt,
); err != nil {
return fmt.Errorf("insert attachment job: %w", err)
}
return nil
}
func insertAttachmentJobOutbox(
ctx context.Context,
tx pgx.Tx,
job model.AttachmentJob,
traceparent string,
tracestate string,
) error {
payload, err := json.Marshal(model.AttachmentJobMessage{
SchemaVersion: 1, JobID: job.ID, Type: "attachment.process.v1",
AttachmentID: job.AttachmentID, PipelineVersion: job.PipelineVersion,
Attempt: job.Attempt, CreatedAt: job.CreatedAt,
Traceparent: traceparent, Tracestate: tracestate,
})
if err != nil {
return fmt.Errorf("marshal attachment job event: %w", err)
}
const query = `
INSERT INTO outbox_events (
id, topic, partition_key, payload, job_id, attempt, available_at
)
VALUES ($1, $2, $3, $4, $5, $6, $7)`
if _, err := tx.Exec(
ctx, query, uuid.NewString(), attachmentJobTopic, job.AttachmentID, payload,
job.ID, job.Attempt, job.AvailableAt,
); err != nil {
return fmt.Errorf("insert attachment job outbox event: %w", err)
}
return nil
}
func (s *Store) AttachmentJob(ctx context.Context, jobID string) (model.AttachmentJob, error) {
const query = `SELECT ` + attachmentJobColumns + ` FROM attachment_jobs WHERE id = $1`
var job model.AttachmentJob
if err := scanAttachmentJob(s.pool.QueryRow(ctx, query, jobID), &job); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.AttachmentJob{}, ErrNotFound
}
return model.AttachmentJob{}, fmt.Errorf("get attachment job: %w", err)
}
return job, nil
}
func (s *Store) ClaimAttachmentJob(
ctx context.Context,
jobID string,
attachmentID string,
pipelineVersion int,
attempt int,
workerID string,
leaseDuration time.Duration,
) (model.AttachmentJobLease, error) {
if leaseDuration <= 0 || workerID == "" {
return model.AttachmentJobLease{}, model.ErrInvalidInput
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.AttachmentJobLease{}, fmt.Errorf("begin claim attachment job: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
if err != nil {
return model.AttachmentJobLease{}, err
}
if attachment.ID != attachmentID || job.AttachmentID != attachmentID {
return model.AttachmentJobLease{}, model.ErrInvalidInput
}
if attachment.PipelineVersion != pipelineVersion || job.PipelineVersion != pipelineVersion {
return model.AttachmentJobLease{}, model.ErrStalePipeline
}
if job.Status == model.AttachmentJobReady || attachment.Status == model.AttachmentReady {
return model.AttachmentJobLease{}, model.ErrJobComplete
}
if job.Attempt != attempt {
return model.AttachmentJobLease{}, model.ErrStaleAttempt
}
if job.Status == model.AttachmentJobProcessing {
return model.AttachmentJobLease{}, model.ErrJobBusy
}
if job.Status != model.AttachmentJobQueued {
return model.AttachmentJobLease{}, ErrConflict
}
if job.AvailableAt.After(time.Now()) {
return model.AttachmentJobLease{}, model.ErrJobNotDue
}
job.LeaseToken = uuid.NewString()
job.LeaseOwner = workerID
leaseUntil := time.Now().UTC().Add(leaseDuration)
job.LeaseUntil = &leaseUntil
job.Status = model.AttachmentJobProcessing
const updateJob = `
UPDATE attachment_jobs
SET status = 'processing', lease_token = $2, lease_owner = $3,
lease_until = $4, error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1`
if _, err := tx.Exec(ctx, updateJob, job.ID, job.LeaseToken, job.LeaseOwner, leaseUntil); err != nil {
return model.AttachmentJobLease{}, fmt.Errorf("claim attachment job: %w", err)
}
const updateAttachment = `
UPDATE attachments
SET status = 'processing', error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2 AND status = 'queued'`
result, err := tx.Exec(ctx, updateAttachment, attachment.ID, pipelineVersion)
if err != nil {
return model.AttachmentJobLease{}, fmt.Errorf("mark attachment processing: %w", err)
}
if result.RowsAffected() != 1 {
return model.AttachmentJobLease{}, model.ErrStalePipeline
}
if err := tx.Commit(ctx); err != nil {
return model.AttachmentJobLease{}, fmt.Errorf("commit attachment job claim: %w", err)
}
attachment.Status = model.AttachmentProcessing
attachment.ErrorCode = nil
return model.AttachmentJobLease{Job: job, Attachment: attachment}, nil
}
func lockAttachmentJob(ctx context.Context, tx pgx.Tx, jobID string) (model.AttachmentJob, model.Attachment, error) {
query := `
SELECT ` + prefixedAttachmentJobColumns("j") + `, ` + attachmentColumnsWithAlias("a") + `
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id
WHERE j.id = $1
FOR UPDATE OF j, a`
var job model.AttachmentJob
var attachment model.Attachment
var leaseToken, leaseOwner *string
var stagingKey *string
err := tx.QueryRow(ctx, query, jobID).Scan(
&job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt,
&leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt,
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
)
if errors.Is(err, pgx.ErrNoRows) {
return model.AttachmentJob{}, model.Attachment{}, ErrNotFound
}
if err != nil {
return model.AttachmentJob{}, model.Attachment{}, fmt.Errorf("lock attachment job: %w", err)
}
if leaseToken != nil {
job.LeaseToken = *leaseToken
}
if leaseOwner != nil {
job.LeaseOwner = *leaseOwner
}
if stagingKey != nil {
attachment.StagingKey = *stagingKey
}
return job, attachment, nil
}
func prefixedAttachmentJobColumns(alias string) string {
return fmt.Sprintf(
"%[1]s.id, %[1]s.attachment_id, %[1]s.pipeline_version, %[1]s.attempt, %[1]s.status, "+
"%[1]s.available_at, %[1]s.lease_token, %[1]s.lease_owner, %[1]s.lease_until, "+
"%[1]s.error_code, %[1]s.created_at, %[1]s.updated_at",
alias,
)
}
func scanAttachmentJob(row rowScanner, job *model.AttachmentJob) error {
var leaseToken, leaseOwner *string
err := row.Scan(
&job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt,
&leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt,
)
if leaseToken != nil {
job.LeaseToken = *leaseToken
}
if leaseOwner != nil {
job.LeaseOwner = *leaseOwner
}
return err
}
func (s *Store) CompleteAttachmentJob(
ctx context.Context,
lease model.AttachmentJobLease,
variants []model.Variant,
) (model.JobMutationResult, error) {
if !validCompletedVariants(variants) {
return "", model.ErrInvalidInput
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return "", fmt.Errorf("begin complete attachment job: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
if err != nil || result != model.JobMutationApplied {
return result, err
}
const deleteVariants = `
DELETE FROM attachment_variants
WHERE attachment_id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, deleteVariants, attachment.ID, job.PipelineVersion); err != nil {
return "", fmt.Errorf("replace attachment variants: %w", err)
}
var originalKey string
const insertVariant = `
INSERT INTO attachment_variants (
attachment_id, pipeline_version, name, content_type, size_bytes,
sha256, width, height, object_key
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`
for _, variant := range variants {
if variant.Name == "original" {
originalKey = variant.ObjectKey
}
if _, err := tx.Exec(
ctx, insertVariant, attachment.ID, job.PipelineVersion, variant.Name,
variant.ContentType, variant.SizeBytes, variant.SHA256, variant.Width,
variant.Height, variant.ObjectKey,
); err != nil {
return "", fmt.Errorf("insert attachment variant %s: %w", variant.Name, err)
}
}
const completeJob = `
UPDATE attachment_jobs
SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND lease_token = $2`
if _, err := tx.Exec(ctx, completeJob, job.ID, lease.Job.LeaseToken); err != nil {
return "", fmt.Errorf("complete attachment job: %w", err)
}
const completeAttachment = `
UPDATE attachments
SET status = 'ready', object_key = $3, ready_at = clock_timestamp(),
staging_key = NULL, error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil {
return "", fmt.Errorf("complete attachment: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return "", fmt.Errorf("commit completed attachment job: %w", err)
}
return model.JobMutationApplied, nil
}
func validCompletedVariants(variants []model.Variant) bool {
seen := make(map[string]struct{}, len(variants))
for _, variant := range variants {
if variant.Name == "" || variant.ObjectKey == "" {
return false
}
if _, duplicate := seen[variant.Name]; duplicate {
return false
}
seen[variant.Name] = struct{}{}
}
_, hasOriginal := seen["original"]
_, hasThumbnail := seen["thumbnail"]
return hasOriginal && hasThumbnail
}
func (s *Store) RetryAttachmentJob(
ctx context.Context,
lease model.AttachmentJobLease,
errorCode string,
nextAttemptAt time.Time,
) (model.JobMutationResult, error) {
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
return "", model.ErrInvalidInput
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return "", fmt.Errorf("begin retry attachment job: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
if err != nil || result != model.JobMutationApplied {
return result, err
}
job.Attempt++
job.Status = model.AttachmentJobQueued
job.AvailableAt = nextAttemptAt.UTC()
job.ErrorCode = &errorCode
const retryJob = `
UPDATE attachment_jobs
SET status = 'queued', attempt = $3, available_at = $4,
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = $5, updated_at = clock_timestamp()
WHERE id = $1 AND lease_token = $2`
if _, err := tx.Exec(
ctx, retryJob, job.ID, lease.Job.LeaseToken, job.Attempt, job.AvailableAt, errorCode,
); err != nil {
return "", fmt.Errorf("retry attachment job: %w", err)
}
const queueAttachment = `
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil {
return "", fmt.Errorf("requeue attachment: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
return "", err
}
if err := tx.Commit(ctx); err != nil {
return "", fmt.Errorf("commit attachment job retry: %w", err)
}
return model.JobMutationApplied, nil
}
func (s *Store) FailAttachmentJob(
ctx context.Context,
lease model.AttachmentJobLease,
errorCode string,
) (model.JobMutationResult, error) {
if !validErrorCode(errorCode) {
return "", model.ErrInvalidInput
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return "", fmt.Errorf("begin fail attachment job: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
if err != nil || result != model.JobMutationApplied {
return result, err
}
const failJob = `
UPDATE attachment_jobs
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = $3, updated_at = clock_timestamp()
WHERE id = $1 AND lease_token = $2`
if _, err := tx.Exec(ctx, failJob, job.ID, lease.Job.LeaseToken, errorCode); err != nil {
return "", fmt.Errorf("fail attachment job: %w", err)
}
const failAttachment = `
UPDATE attachments
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil {
return "", fmt.Errorf("fail attachment: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return "", fmt.Errorf("commit failed attachment job: %w", err)
}
return model.JobMutationApplied, nil
}
func lockJobMutation(
ctx context.Context,
tx pgx.Tx,
lease model.AttachmentJobLease,
) (model.AttachmentJob, model.Attachment, model.JobMutationResult, error) {
job, attachment, err := lockAttachmentJob(ctx, tx, lease.Job.ID)
if err != nil {
return model.AttachmentJob{}, model.Attachment{}, "", err
}
if attachment.PipelineVersion != lease.Job.PipelineVersion || job.PipelineVersion != lease.Job.PipelineVersion {
return job, attachment, model.JobMutationStale, nil
}
if job.Status == model.AttachmentJobReady && attachment.Status == model.AttachmentReady {
return job, attachment, model.JobMutationAlreadyComplete, nil
}
if job.PipelineVersion != lease.Job.PipelineVersion ||
job.Attempt != lease.Job.Attempt ||
job.Status != model.AttachmentJobProcessing ||
job.LeaseToken == "" || job.LeaseToken != lease.Job.LeaseToken ||
job.LeaseUntil == nil || !job.LeaseUntil.After(time.Now()) {
return job, attachment, model.JobMutationStale, nil
}
return job, attachment, model.JobMutationApplied, nil
}
func validErrorCode(errorCode string) bool {
return len(errorCode) >= 1 && len(errorCode) <= 64
}
+210
View File
@@ -0,0 +1,210 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
)
func (s *Store) RecoverExpiredAttachmentJobs(
ctx context.Context,
limit int,
maxAttempts int,
retryAt time.Time,
) (model.RecoveryResult, error) {
if limit < 1 || limit > 1000 || maxAttempts < 1 {
return model.RecoveryResult{}, model.ErrInvalidInput
}
if retryAt.IsZero() {
retryAt = time.Now().UTC()
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
const selectExpired = `
SELECT id
FROM attachment_jobs
WHERE status = 'processing' AND lease_until <= clock_timestamp()
ORDER BY lease_until, id
LIMIT $1
FOR UPDATE SKIP LOCKED`
rows, err := tx.Query(ctx, selectExpired, limit)
if err != nil {
return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err)
}
jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err)
}
var recovered model.RecoveryResult
for _, jobID := range jobIDs {
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
if err != nil {
return model.RecoveryResult{}, err
}
if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) {
continue
}
if job.PipelineVersion != attachment.PipelineVersion {
if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil {
return model.RecoveryResult{}, err
}
recovered.Failed++
continue
}
if job.Attempt >= maxAttempts {
if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil {
return model.RecoveryResult{}, err
}
recovered.Failed++
continue
}
job.Attempt++
job.Status = model.AttachmentJobQueued
job.AvailableAt = retryAt.UTC()
const requeueJob = `
UPDATE attachment_jobs
SET status = 'queued', attempt = $2, available_at = $3,
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = 'lease_expired', updated_at = clock_timestamp()
WHERE id = $1`
if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err)
}
const requeueAttachment = `
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
return model.RecoveryResult{}, err
}
recovered.Requeued++
}
if err := tx.Commit(ctx); err != nil {
return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err)
}
return recovered, nil
}
func markExpiredJobFailed(
ctx context.Context,
tx pgx.Tx,
job model.AttachmentJob,
attachment model.Attachment,
errorCode string,
failAttachment bool,
) error {
const failJob = `
UPDATE attachment_jobs
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = $2, updated_at = clock_timestamp()
WHERE id = $1`
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
return fmt.Errorf("fail expired attachment job: %w", err)
}
if !failAttachment {
return nil
}
const failAttachmentQuery = `
UPDATE attachments
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil {
return fmt.Errorf("fail attachment after expired lease: %w", err)
}
return nil
}
func (s *Store) ReplayFailedAttachment(
ctx context.Context,
attachmentID string,
actor string,
reason string,
) (model.AttachmentJob, error) {
if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 {
return model.AttachmentJob{}, model.ErrInvalidInput
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
const selectJob = `
SELECT j.id
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed'
FOR UPDATE OF j, a`
var jobID string
if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.AttachmentJob{}, ErrConflict
}
return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err)
}
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
if err != nil {
return model.AttachmentJob{}, err
}
previousAttempt := job.Attempt
job.Attempt++
job.Status = model.AttachmentJobQueued
job.AvailableAt = time.Now().UTC()
job.ErrorCode = nil
job.LeaseToken = ""
job.LeaseOwner = ""
job.LeaseUntil = nil
const replayJob = `
UPDATE attachment_jobs
SET status = 'queued', attempt = $2, available_at = $3,
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1`
if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err)
}
const replayAttachment = `
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil {
return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err)
}
const insertAudit = `
INSERT INTO attachment_replay_audit (
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason
)
VALUES ($1, $2, $3, $4, $5, $6, $7)`
if _, err := tx.Exec(
ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID,
previousAttempt, job.Attempt, actor, reason,
); err != nil {
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
return model.AttachmentJob{}, err
}
if err := tx.Commit(ctx); err != nil {
return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err)
}
return job, nil
}
+90 -7
View File
@@ -13,7 +13,7 @@ import (
const attachmentColumns = `
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, created_at, updated_at, ready_at`
object_key, status, created_at, updated_at, ready_at, pipeline_version, error_code, staging_key`
type rowScanner interface {
Scan(...any) error
@@ -74,7 +74,7 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri
SELECT ` + attachmentColumnsWithAlias("a") + `
FROM attachments a
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
WHERE a.id = $1 AND a.status = 'ready'`
WHERE a.id = $1 AND a.status IN ('queued', 'processing', 'ready', 'failed')`
var attachment model.Attachment
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment)
@@ -84,9 +84,68 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri
if err != nil {
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
}
if attachment.Status == model.AttachmentReady {
attachment.Variants, err = s.attachmentVariants(ctx, attachment.ID, attachment.PipelineVersion)
if err != nil {
return model.Attachment{}, err
}
}
return publicAttachment(attachment), nil
}
func (s *Store) AttachmentVariantForUser(
ctx context.Context,
attachmentID string,
userID string,
variantName string,
) (model.Attachment, model.Variant, error) {
attachment, err := s.AttachmentForUser(ctx, attachmentID, userID)
if err != nil {
return model.Attachment{}, model.Variant{}, err
}
if attachment.Status != model.AttachmentReady {
return model.Attachment{}, model.Variant{}, ErrConflict
}
for _, variant := range attachment.Variants {
if variant.Name == variantName {
return attachment, variant, nil
}
}
if variantName == "original" && attachment.ObjectKey != "" {
return attachment, model.Variant{
Name: "original", ContentType: attachment.ContentType, SizeBytes: attachment.SizeBytes,
SHA256: attachment.SHA256, ObjectKey: attachment.ObjectKey,
PipelineVersion: attachment.PipelineVersion,
}, nil
}
return model.Attachment{}, model.Variant{}, ErrNotFound
}
func (s *Store) attachmentVariants(ctx context.Context, attachmentID string, pipelineVersion int) ([]model.Variant, error) {
const query = `
SELECT name, content_type, size_bytes, sha256, width, height, object_key, pipeline_version
FROM attachment_variants
WHERE attachment_id = $1 AND pipeline_version = $2
ORDER BY CASE name
WHEN 'original' THEN 0
WHEN 'thumbnail' THEN 1
WHEN 'crop' THEN 2
WHEN 'optimized' THEN 3
ELSE 4
END`
rows, err := s.pool.Query(ctx, query, attachmentID, pipelineVersion)
if err != nil {
return nil, fmt.Errorf("list attachment variants: %w", err)
}
defer rows.Close()
variants, err := pgx.CollectRows(rows, pgx.RowToStructByPos[model.Variant])
if err != nil {
return nil, fmt.Errorf("scan attachment variants: %w", err)
}
return variants, nil
}
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2`
@@ -102,7 +161,12 @@ func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey
}
func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`
const query = `
SELECT EXISTS (
SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1
UNION ALL
SELECT 1 FROM attachment_variants WHERE object_key = $1
)`
var exists bool
if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil {
return false, fmt.Errorf("check attachment by object key: %w", err)
@@ -144,12 +208,18 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi
}
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
return row.Scan(
var stagingKey *string
err := row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
&attachment.Referenced,
)
if stagingKey != nil {
attachment.StagingKey = *stagingKey
}
return err
}
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
@@ -232,23 +302,36 @@ func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID str
}
func scanAttachment(row rowScanner, attachment *model.Attachment) error {
return row.Scan(
var stagingKey *string
err := row.Scan(
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
)
if stagingKey != nil {
attachment.StagingKey = *stagingKey
}
return err
}
func attachmentColumnsWithAlias(alias string) string {
return fmt.Sprintf(
"%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+
"%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at",
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at, %[1]s.pipeline_version, "+
"%[1]s.error_code, %[1]s.staging_key",
alias,
)
}
func publicAttachment(attachment model.Attachment) model.Attachment {
attachment.Variants = []model.Variant{}
variants := make([]model.Variant, 0, len(attachment.Variants))
for _, variant := range attachment.Variants {
if variant.Name != "original" {
variants = append(variants, variant)
}
}
attachment.Variants = variants
return attachment
}
+6
View File
@@ -8,4 +8,10 @@ var (
ErrInvalidMembers = model.ErrInvalidMembers
ErrInvalidCursor = model.ErrInvalidCursor
ErrIdempotencyConflict = model.ErrIdempotencyConflict
ErrStaleLease = model.ErrStaleLease
ErrStaleAttempt = model.ErrStaleAttempt
ErrStalePipeline = model.ErrStalePipeline
ErrJobBusy = model.ErrJobBusy
ErrJobComplete = model.ErrJobComplete
ErrJobNotDue = model.ErrJobNotDue
)
+19 -9
View File
@@ -51,13 +51,16 @@ func (s *Store) CreateMessage(
if !visible {
return model.Message{}, false, ErrNotFound
}
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
validAttachments, failedAttachment, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
if err != nil {
return model.Message{}, false, err
}
if !validAttachments {
return model.Message{}, false, ErrNotFound
}
if failedAttachment {
return model.Message{}, false, ErrConflict
}
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
return model.Message{}, false, err
}
@@ -92,28 +95,35 @@ func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte,
return sha256.Sum256(payload), nil
}
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, bool, error) {
if len(attachmentIDs) == 0 {
return true, nil
return true, false, nil
}
const query = `
SELECT id
SELECT id, status
FROM attachments
WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready'
WHERE chat_id = $1 AND id::text = ANY($2)
AND status IN ('queued', 'processing', 'ready', 'failed')
FOR UPDATE`
rows, err := tx.Query(ctx, query, chatID, attachmentIDs)
if err != nil {
return false, fmt.Errorf("validate message attachments: %w", err)
return false, false, fmt.Errorf("validate message attachments: %w", err)
}
defer rows.Close()
count := 0
var count int
var failed bool
for rows.Next() {
var id, status string
if err := rows.Scan(&id, &status); err != nil {
return false, false, fmt.Errorf("scan message attachment: %w", err)
}
count++
failed = failed || status == model.AttachmentFailed
}
if err := rows.Err(); err != nil {
return false, fmt.Errorf("validate message attachments: %w", err)
return false, false, fmt.Errorf("validate message attachments: %w", err)
}
return count == len(attachmentIDs), nil
return count == len(attachmentIDs), failed, nil
}
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
+113
View File
@@ -0,0 +1,113 @@
package postgres
import (
"context"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
)
func (s *Store) ClaimOutboxEvents(
ctx context.Context,
limit int,
leaseDuration time.Duration,
) ([]model.OutboxEvent, error) {
if limit < 1 || limit > 1000 || leaseDuration <= 0 {
return nil, model.ErrInvalidInput
}
claimToken := uuid.NewString()
query := `
WITH candidates AS (
SELECT id
FROM outbox_events
WHERE published_at IS NULL
AND available_at <= clock_timestamp()
AND (claim_until IS NULL OR claim_until <= clock_timestamp())
ORDER BY available_at, id
LIMIT $1
FOR UPDATE SKIP LOCKED
)
UPDATE outbox_events event
SET claim_token = $2,
claim_until = clock_timestamp() + $3 * INTERVAL '1 millisecond',
publish_tries = event.publish_tries + 1
FROM candidates
WHERE event.id = candidates.id
RETURNING event.id, event.topic, event.partition_key, event.payload, event.job_id,
event.attempt, event.available_at, event.publish_tries, event.claim_token,
event.claim_until, event.last_error_code, event.created_at`
rows, err := s.pool.Query(ctx, query, limit, claimToken, leaseDuration.Milliseconds())
if err != nil {
return nil, fmt.Errorf("claim outbox events: %w", err)
}
defer rows.Close()
events, err := pgx.CollectRows(rows, pgx.RowToFunc[model.OutboxEvent](func(row pgx.CollectableRow) (model.OutboxEvent, error) {
var event model.OutboxEvent
err := row.Scan(
&event.ID, &event.Topic, &event.PartitionKey, &event.Payload, &event.JobID,
&event.Attempt, &event.AvailableAt, &event.PublishTries, &event.LeaseToken,
&event.LeaseUntil, &event.LastErrorCode, &event.CreatedAt,
)
return event, err
}))
if err != nil {
return nil, fmt.Errorf("scan claimed outbox events: %w", err)
}
return events, nil
}
func (s *Store) MarkOutboxPublished(ctx context.Context, eventID, leaseToken string) (bool, error) {
const query = `
UPDATE outbox_events
SET published_at = clock_timestamp(), claim_token = NULL, claim_until = NULL,
last_error_code = NULL
WHERE id = $1 AND claim_token = $2 AND published_at IS NULL
AND claim_until > clock_timestamp()`
result, err := s.pool.Exec(ctx, query, eventID, leaseToken)
if err != nil {
return false, fmt.Errorf("mark outbox event published: %w", err)
}
return result.RowsAffected() == 1, nil
}
func (s *Store) ReleaseOutboxEvent(
ctx context.Context,
eventID string,
leaseToken string,
errorCode string,
nextAttemptAt time.Time,
) (bool, error) {
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
return false, model.ErrInvalidInput
}
const query = `
UPDATE outbox_events
SET available_at = $3, claim_token = NULL, claim_until = NULL, last_error_code = $4
WHERE id = $1 AND claim_token = $2 AND published_at IS NULL
AND claim_until > clock_timestamp()`
result, err := s.pool.Exec(ctx, query, eventID, leaseToken, nextAttemptAt.UTC(), errorCode)
if err != nil {
return false, fmt.Errorf("release outbox event: %w", err)
}
return result.RowsAffected() == 1, nil
}
func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (int, error) {
if now.IsZero() {
now = time.Now().UTC()
}
const query = `
UPDATE outbox_events
SET claim_token = NULL, claim_until = NULL
WHERE published_at IS NULL AND claim_until <= $1`
result, err := s.pool.Exec(ctx, query, now.UTC())
if err != nil {
return 0, fmt.Errorf("recover expired outbox events: %w", err)
}
return int(result.RowsAffected()), nil
}
+189
View File
@@ -2,6 +2,7 @@ package postgres_test
import (
"context"
"encoding/json"
"errors"
"os"
"strings"
@@ -159,6 +160,165 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
s.False(deleted)
}
func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
s.Equal(model.AttachmentQueued, attachment.Status)
events, err := s.store.ClaimOutboxEvents(context.Background(), 10, time.Minute)
s.Require().NoError(err)
s.Require().NotEmpty(events)
var event model.OutboxEvent
for _, candidate := range events {
if candidate.JobID == job.ID {
event = candidate
break
}
}
s.Require().NotEmpty(event.ID)
var message model.AttachmentJobMessage
s.Require().NoError(json.Unmarshal(event.Payload, &message))
s.Equal(job.ID, message.JobID)
s.Equal(attachment.ID, message.AttachmentID)
s.Equal(1, message.Attempt)
published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken)
s.Require().NoError(err)
s.True(published)
_, replay, err := s.store.CreateMessage(
context.Background(), chatID, owner.ID, "queued attachment",
[]string{attachment.ID}, uuid.NewString(), 24*time.Hour,
)
s.Require().NoError(err)
s.False(replay)
_, err = s.store.ClaimAttachmentJob(
context.Background(), job.ID, uuid.NewString(), job.PipelineVersion, job.Attempt, "worker-invalid", time.Minute,
)
s.ErrorIs(err, model.ErrInvalidInput)
lease, err := s.store.ClaimAttachmentJob(
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
)
s.Require().NoError(err)
s.Equal(model.AttachmentProcessing, lease.Attachment.Status)
s.NotEmpty(lease.Attachment.StagingKey)
variants := []model.Variant{
{
Name: "original", ContentType: "image/png", SizeBytes: 128,
SHA256: strings.Repeat("a", 64), Width: 320, Height: 200,
ObjectKey: "attachments/" + attachment.ID + "/v1/original.png",
},
{
Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64,
SHA256: strings.Repeat("b", 64), Width: 256, Height: 160,
ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg",
},
}
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants)
s.Require().NoError(err)
s.Equal(model.JobMutationApplied, result)
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
s.Require().NoError(err)
s.Equal(model.JobMutationAlreadyComplete, result)
metadata, variant, err := s.store.AttachmentVariantForUser(
context.Background(), attachment.ID, owner.ID, "thumbnail",
)
s.Require().NoError(err)
s.Equal(model.AttachmentReady, metadata.Status)
s.Len(metadata.Variants, 1)
s.Equal(variants[1].ObjectKey, variant.ObjectKey)
v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2}
queuedV2, err := s.store.QueueAttachmentPipelineVersion(
context.Background(), attachment.ID, v2, "", "",
)
s.Require().NoError(err)
s.Equal(model.AttachmentQueued, queuedV2.Status)
s.Equal(2, queuedV2.PipelineVersion)
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
s.Require().NoError(err)
s.Equal(model.JobMutationStale, result)
}
func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
lease, err := s.store.ClaimAttachmentJob(
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
)
s.Require().NoError(err)
result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image")
s.Require().NoError(err)
s.Equal(model.JobMutationApplied, result)
_, _, err = s.store.CreateMessage(
context.Background(), chatID, owner.ID, "failed attachment",
[]string{attachment.ID}, uuid.NewString(), 24*time.Hour,
)
s.ErrorIs(err, postgres.ErrConflict)
metadata, err := s.store.AttachmentForUser(context.Background(), attachment.ID, owner.ID)
s.Require().NoError(err)
s.Equal(model.AttachmentFailed, metadata.Status)
s.Require().NotNil(metadata.ErrorCode)
s.Equal("invalid_image", *metadata.ErrorCode)
replayed, err := s.store.ReplayFailedAttachment(
context.Background(), attachment.ID, "operator@example", "input repaired",
)
s.Require().NoError(err)
s.Equal(2, replayed.Attempt)
s.Equal(model.AttachmentJobQueued, replayed.Status)
var auditCount int
err = s.pool.QueryRow(
context.Background(), `SELECT count(*) FROM attachment_replay_audit WHERE job_id = $1`, job.ID,
).Scan(&auditCount)
s.Require().NoError(err)
s.Equal(1, auditCount)
}
func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
lease, err := s.store.ClaimAttachmentJob(
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
)
s.Require().NoError(err)
_, err = s.pool.Exec(
context.Background(),
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
job.ID,
)
s.Require().NoError(err)
recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 5, time.Now())
s.Require().NoError(err)
s.Equal(1, recovered.Requeued)
s.Zero(recovered.Failed)
current, err := s.store.AttachmentJob(context.Background(), job.ID)
s.Require().NoError(err)
s.Equal(2, current.Attempt)
s.Equal(model.AttachmentJobQueued, current.Status)
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, []model.Variant{
{
Name: "original", ContentType: "image/png", SizeBytes: 128,
SHA256: strings.Repeat("a", 64), Width: 320, Height: 200,
ObjectKey: "attachments/" + attachment.ID + "/v1/original.png",
},
{
Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64,
SHA256: strings.Repeat("b", 64), Width: 256, Height: 160,
ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg",
},
})
s.Require().NoError(err)
s.Equal(model.JobMutationStale, result)
}
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
@@ -344,6 +504,35 @@ func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string
return owner, chat.ID
}
func createQueuedAttachment(
t *testing.T,
store *postgres.Store,
ownerID string,
chatID string,
) (model.Attachment, model.AttachmentJob) {
t.Helper()
attachmentID := uuid.NewString()
stagingKey := "staging/" + attachmentID + "/original"
pending, err := store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: attachmentID, ChatID: chatID, OwnerID: ownerID, Filename: "photo.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
ObjectKey: stagingKey,
})
if err != nil {
t.Fatalf("CreatePendingAttachment() error = %v", err)
}
job := model.AttachmentJob{
ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, Attempt: 1,
}
queued, err := store.QueueAttachmentWithJobAndOutbox(
context.Background(), pending.ID, stagingKey, job, "", "",
)
if err != nil {
t.Fatalf("QueueAttachmentWithJobAndOutbox() error = %v", err)
}
return queued, job
}
func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int {
t.Helper()
var count int
+40
View File
@@ -23,6 +23,7 @@ type ObjectStore interface {
type Config struct {
Prefix string
OrphanPrefixes []string
PageSize int
GraceTime time.Duration
UnreferencedRetention time.Duration
@@ -65,6 +66,11 @@ func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconcile
if config.UnreferencedRetention <= 0 {
return nil, errors.New("unreferenced retention must be positive")
}
for _, prefix := range config.OrphanPrefixes {
if prefix == "" || prefix == config.Prefix {
return nil, errors.New("orphan prefix must be non-empty and distinct")
}
}
if config.Now == nil {
config.Now = time.Now
}
@@ -113,9 +119,35 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) {
}
}
}
for _, prefix := range r.config.OrphanPrefixes {
if err := r.scanOrphanPrefix(ctx, prefix, anomalyCutoff, &report); err != nil {
return report, err
}
}
return report, nil
}
func (r *Reconciler) scanOrphanPrefix(
ctx context.Context,
prefix string,
cutoff time.Time,
report *Report,
) error {
objects, err := newObjectIterator(ctx, r.objects, prefix, r.config.PageSize)
if err != nil {
return err
}
for objects.current != nil {
if err := r.handleOrphanObject(ctx, *objects.current, cutoff, report); err != nil {
return err
}
if err := objects.advance(ctx); err != nil {
return err
}
}
return nil
}
func (r *Reconciler) handlePair(
ctx context.Context,
metadata model.Attachment,
@@ -226,6 +258,14 @@ func (r *Reconciler) handleMissingObject(
report *Report,
) error {
report.MetadataScanned++
if metadata.Status == model.AttachmentPending {
if metadata.CreatedAt.After(cutoff) {
report.FreshPendingAttachments++
return nil
}
report.OldPendingAttachments++
return r.deletePair(ctx, metadata, cutoff, report)
}
if metadata.Status == model.AttachmentDeleting {
return r.deleteClaimedMetadata(ctx, metadata, report)
}
+11 -4
View File
@@ -21,6 +21,7 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) {
"attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false),
"attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false),
"attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false),
"staging/pending/original": attachment("pending", "staging/pending/original", model.AttachmentPending, old, false),
}}
objects := &objectStub{items: map[string]model.StoredObject{
"attachments/a/original": {Key: "attachments/a/original", LastModified: old},
@@ -28,27 +29,33 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) {
"attachments/c/original": {Key: "attachments/c/original", LastModified: old},
"attachments/d/original": {Key: "attachments/d/original", LastModified: fresh},
"attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old},
"staging/orphan/original": {Key: "staging/orphan/original", LastModified: old},
"staging/pending/original": {Key: "staging/pending/original", LastModified: old},
}}
reconciler, err := New(metadata, objects, Config{
Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour,
OrphanPrefixes: []string{"staging/"},
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
})
require.NoError(t, err)
report, err := reconciler.Run(context.Background())
require.NoError(t, err)
require.Equal(t, 1, report.OldPendingAttachments)
require.Equal(t, 2, report.OldPendingAttachments)
require.Equal(t, 1, report.OldUnreferencedAttachments)
require.Equal(t, 1, report.FreshUnreferencedAttachments)
require.Equal(t, 1, report.DeletedOrphanObjects)
require.Equal(t, 2, report.DeletedAttachmentObjects)
require.Equal(t, 3, report.DeletedUnreferencedRows)
require.Equal(t, 2, report.DeletedOrphanObjects)
require.Equal(t, 3, report.DeletedAttachmentObjects)
require.Equal(t, 4, report.DeletedUnreferencedRows)
require.Contains(t, metadata.items, "attachments/b/original")
require.Contains(t, metadata.items, "attachments/d/original")
require.NotContains(t, metadata.items, "attachments/a/original")
require.NotContains(t, metadata.items, "attachments/c/original")
require.NotContains(t, metadata.items, "attachments/missing/original")
require.NotContains(t, metadata.items, "staging/pending/original")
require.NotContains(t, objects.items, "attachments/orphan/original")
require.NotContains(t, objects.items, "staging/orphan/original")
require.NotContains(t, objects.items, "staging/pending/original")
repeated, err := reconciler.Run(context.Background())
require.NoError(t, err)
@@ -0,0 +1,135 @@
ALTER TABLE attachments
DROP CONSTRAINT attachments_status,
DROP CONSTRAINT attachments_ready_at;
ALTER TABLE attachments
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1,
ADD COLUMN error_code TEXT,
ADD COLUMN staging_key TEXT;
UPDATE attachments
SET staging_key = object_key
WHERE status = 'pending';
ALTER TABLE attachments
ADD CONSTRAINT attachments_status CHECK (
status IN ('pending', 'queued', 'processing', 'ready', 'failed', 'deleting')
),
ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1),
ADD CONSTRAINT attachments_error_code CHECK (
(status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR
(status <> 'failed' AND error_code IS NULL)
),
ADD CONSTRAINT attachments_ready_at CHECK (
(status = 'ready' AND ready_at IS NOT NULL) OR
(status <> 'ready')
),
ADD CONSTRAINT attachments_staging_key_length CHECK (
staging_key IS NULL OR char_length(staging_key) BETWEEN 1 AND 1024
);
CREATE INDEX attachments_status_updated_at_idx
ON attachments (status, updated_at, id);
CREATE TABLE attachment_jobs (
id UUID PRIMARY KEY,
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL,
attempt INTEGER NOT NULL DEFAULT 1,
status TEXT NOT NULL DEFAULT 'queued',
available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
lease_token UUID,
lease_owner TEXT,
lease_until TIMESTAMPTZ,
error_code TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version),
CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1),
CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1),
CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT attachment_jobs_lease CHECK (
(status = 'processing' AND lease_token IS NOT NULL AND lease_owner IS NOT NULL AND lease_until IS NOT NULL) OR
(status <> 'processing' AND lease_token IS NULL AND lease_owner IS NULL AND lease_until IS NULL)
),
CONSTRAINT attachment_jobs_error_code CHECK (
error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64
)
);
CREATE INDEX attachment_jobs_due_idx
ON attachment_jobs (available_at, id)
WHERE status = 'queued';
CREATE INDEX attachment_jobs_expired_lease_idx
ON attachment_jobs (lease_until, id)
WHERE status = 'processing';
CREATE TABLE attachment_variants (
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version INTEGER NOT NULL,
name TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
sha256 TEXT NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
object_key TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachment_variants_pk PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT attachment_variants_object_key_unique UNIQUE (object_key),
CONSTRAINT attachment_variants_pipeline_version CHECK (pipeline_version >= 1),
CONSTRAINT attachment_variants_name CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')),
CONSTRAINT attachment_variants_content_type CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')),
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1),
CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachment_variants_dimensions CHECK (width >= 1 AND height >= 1),
CONSTRAINT attachment_variants_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
);
CREATE TABLE outbox_events (
id UUID PRIMARY KEY,
topic TEXT NOT NULL,
partition_key TEXT NOT NULL,
payload JSONB NOT NULL,
job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
attempt INTEGER NOT NULL,
available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
publish_tries INTEGER NOT NULL DEFAULT 0,
claim_token UUID,
claim_until TIMESTAMPTZ,
last_error_code TEXT,
published_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT outbox_events_job_attempt_unique UNIQUE (job_id, attempt),
CONSTRAINT outbox_events_attempt CHECK (attempt >= 1),
CONSTRAINT outbox_events_publish_tries CHECK (publish_tries >= 0),
CONSTRAINT outbox_events_claim CHECK (
(claim_token IS NULL AND claim_until IS NULL) OR
(claim_token IS NOT NULL AND claim_until IS NOT NULL)
),
CONSTRAINT outbox_events_last_error_code CHECK (
last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64
)
);
CREATE INDEX outbox_events_due_idx
ON outbox_events (available_at, id)
WHERE published_at IS NULL;
CREATE TABLE attachment_replay_audit (
id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
previous_attempt INTEGER NOT NULL,
new_attempt INTEGER NOT NULL,
actor TEXT NOT NULL,
reason TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachment_replay_attempts CHECK (previous_attempt >= 1 AND new_attempt = previous_attempt + 1),
CONSTRAINT attachment_replay_actor_length CHECK (char_length(actor) BETWEEN 1 AND 128),
CONSTRAINT attachment_replay_reason_length CHECK (char_length(reason) BETWEEN 1 AND 512)
);
CREATE INDEX attachment_replay_audit_job_created_at_idx
ON attachment_replay_audit (job_id, created_at);