forked from mtusi2026/mtusi2026lab5
feat: add durable image processing pipeline
This commit is contained in:
+10
-1
@@ -24,8 +24,17 @@ func run(logger *slog.Logger, arguments []string) error {
|
||||
if len(arguments) == 1 && arguments[0] == "migrate" {
|
||||
return bootstrap.RunMigrations()
|
||||
}
|
||||
if len(arguments) == 1 && arguments[0] == "dispatch" {
|
||||
return bootstrap.RunDispatcher(logger)
|
||||
}
|
||||
if len(arguments) == 1 && arguments[0] == "worker" {
|
||||
return bootstrap.RunWorker(logger)
|
||||
}
|
||||
if arguments[0] == "reconcile-attachments" {
|
||||
return bootstrap.RunReconciliation(arguments[1:], os.Stdout)
|
||||
}
|
||||
return errors.New("usage: messenger [migrate|reconcile-attachments]")
|
||||
if arguments[0] == "replay-attachment" {
|
||||
return bootstrap.RunAttachmentReplay(arguments[1:], os.Stdout)
|
||||
}
|
||||
return errors.New("usage: messenger [migrate|dispatch|worker|reconcile-attachments|replay-attachment]")
|
||||
}
|
||||
|
||||
@@ -3,10 +3,12 @@ module gitea.privcloud.tech/dizi/mtusi2026lab5
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/deepteams/webp v1.2.8
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.0.99
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/valkey-io/valkey-go v1.0.78
|
||||
@@ -18,6 +20,7 @@ require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deepteams/webp v1.2.8 h1:iFLBxo4V34ixqay6+MTB18zujCM8VSe1ejsYh+ue1nA=
|
||||
github.com/deepteams/webp v1.2.8/go.mod h1:J8Ap+HAixxpKKRN9IpEeSKlfvhsef1v43jKTO7m3f4c=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
@@ -64,6 +72,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/pipeline"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
)
|
||||
|
||||
func RunDispatcher(logger *slog.Logger) error {
|
||||
settings, err := config.LoadDispatcher()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
client, err := newRedisClient(settings.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher",
|
||||
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength,
|
||||
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.Ensure(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{
|
||||
BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration,
|
||||
PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval,
|
||||
RetryDelay: settings.RetryDelay,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("outbox dispatcher started", "stream", settings.Stream)
|
||||
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run)
|
||||
}
|
||||
|
||||
func RunWorker(logger *slog.Logger) error {
|
||||
settings, err := config.LoadWorker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), workerDatabaseConfig(settings))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := newRedisClient(settings.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
consumer := hostname + "-" + uuid.NewString()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
||||
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock,
|
||||
ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := broker.Ensure(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metrics := pipeline.NewMetrics()
|
||||
processorWorker, err := worker.New(
|
||||
logger, postgres.New(pool), broker, objects, processor, metrics,
|
||||
worker.Config{
|
||||
WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration,
|
||||
ProcessingTimeout: settings.ProcessingTimeout,
|
||||
ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts,
|
||||
RetryBase: settings.RetryBase, RetryMax: settings.RetryMax,
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
MemoryBytes: settings.MemoryBytes,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency)
|
||||
return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run)
|
||||
}
|
||||
|
||||
func newRedisClient(rawURL string) (*redis.Client, error) {
|
||||
options, err := redis.ParseURL(rawURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := redis.NewClient(options)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
_ = client.Close()
|
||||
return nil, err
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func runPipelineProcess(
|
||||
logger *slog.Logger,
|
||||
metricsAddress string,
|
||||
metricsHandler http.Handler,
|
||||
run func(context.Context) error,
|
||||
) error {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", metricsHandler)
|
||||
mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
||||
server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
||||
listener, err := net.Listen("tcp", metricsAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serveResult := make(chan error, 1)
|
||||
go func() { serveResult <- server.Serve(listener) }()
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
runResult := make(chan error, 1)
|
||||
go func() { runResult <- run(ctx) }()
|
||||
|
||||
var result error
|
||||
select {
|
||||
case result = <-runResult:
|
||||
stop()
|
||||
case result = <-serveResult:
|
||||
stop()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdownCtx); err != nil && result == nil {
|
||||
result = err
|
||||
}
|
||||
if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) {
|
||||
return nil
|
||||
}
|
||||
logger.Info("pipeline process stopped")
|
||||
return result
|
||||
}
|
||||
|
||||
func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config {
|
||||
return database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func workerDatabaseConfig(settings config.WorkerConfig) database.Config {
|
||||
return database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,7 @@ func RunReconciliation(arguments []string, output io.Writer) error {
|
||||
}
|
||||
reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod,
|
||||
OrphanPrefixes: []string{"staging/"},
|
||||
UnreferencedRetention: *retention, Apply: *apply,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
)
|
||||
|
||||
func RunAttachmentReplay(arguments []string, output io.Writer) error {
|
||||
if len(arguments) != 3 {
|
||||
return errors.New("usage: messenger replay-attachment <attachment-id> <actor> <reason>")
|
||||
}
|
||||
settings, err := config.LoadJobAdmin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(context.Background(), database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
job, err := postgres.New(pool).ReplayFailedAttachment(
|
||||
context.Background(), arguments[0], arguments[1], arguments[2],
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintf(output, "replayed attachment=%s job=%s attempt=%d\n", job.AttachmentID, job.ID, job.Attempt)
|
||||
return err
|
||||
}
|
||||
@@ -54,7 +54,7 @@ func RunServer(logger *slog.Logger) error {
|
||||
attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory,
|
||||
CleanupTimeout: settings.RequestTimeout,
|
||||
CleanupTimeout: settings.RequestTimeout, PipelineVersion: settings.PipelineVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
)
|
||||
|
||||
const payloadField = "payload"
|
||||
|
||||
var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
|
||||
|
||||
var ErrDeliveryNotOwned = errors.New("redis stream delivery is not owned by this consumer")
|
||||
|
||||
var _ outbox.Publisher = (*Broker)(nil)
|
||||
|
||||
type Config struct {
|
||||
Stream string
|
||||
Group string
|
||||
Consumer string
|
||||
FailedStream string
|
||||
FailedMaxLen int64
|
||||
Block time.Duration
|
||||
ClaimMinIdle time.Duration
|
||||
ReadCount int64
|
||||
}
|
||||
|
||||
type Broker struct {
|
||||
client redis.UniversalClient
|
||||
config Config
|
||||
}
|
||||
|
||||
type Delivery struct {
|
||||
ID string
|
||||
Envelope Envelope
|
||||
DecodeError error
|
||||
|
||||
broker *Broker
|
||||
payload []byte
|
||||
}
|
||||
|
||||
type FailedDelivery struct {
|
||||
Code string
|
||||
FailedAt time.Time
|
||||
}
|
||||
|
||||
func New(client redis.UniversalClient, config Config) (*Broker, error) {
|
||||
if client == nil {
|
||||
return nil, errors.New("redis client is required")
|
||||
}
|
||||
if err := config.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Broker{client: client, config: config}, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
switch {
|
||||
case c.Stream == "":
|
||||
return errors.New("stream is required")
|
||||
case c.Group == "":
|
||||
return errors.New("consumer group is required")
|
||||
case c.Consumer == "":
|
||||
return errors.New("consumer name is required")
|
||||
case c.FailedStream == "":
|
||||
return errors.New("failed stream is required")
|
||||
case c.FailedStream == c.Stream:
|
||||
return errors.New("failed stream must differ from job stream")
|
||||
case c.FailedMaxLen <= 0:
|
||||
return errors.New("failed stream maximum length must be positive")
|
||||
case c.Block < 0:
|
||||
return errors.New("block duration must not be negative")
|
||||
case c.ClaimMinIdle <= 0:
|
||||
return errors.New("claim minimum idle time must be positive")
|
||||
case c.ReadCount <= 0:
|
||||
return errors.New("read count must be positive")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Broker) Ensure(ctx context.Context) error {
|
||||
err := b.client.XGroupCreateMkStream(ctx, b.config.Stream, b.config.Group, "0").Err()
|
||||
if err != nil && !isBusyGroup(err) {
|
||||
return fmt.Errorf("create redis stream consumer group: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isBusyGroup(err error) bool {
|
||||
return err != nil && strings.HasPrefix(err.Error(), "BUSYGROUP")
|
||||
}
|
||||
|
||||
func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) {
|
||||
envelope, err := DecodeEnvelope(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
payload, err = EncodeEnvelope(envelope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
id, err := b.client.XAdd(ctx, &redis.XAddArgs{
|
||||
Stream: b.config.Stream,
|
||||
Values: map[string]any{payloadField: string(payload)},
|
||||
}).Result()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("publish redis stream job: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Read(ctx context.Context) ([]Delivery, error) {
|
||||
streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: b.config.Group,
|
||||
Consumer: b.config.Consumer,
|
||||
Streams: []string{b.config.Stream, ">"},
|
||||
Count: b.config.ReadCount,
|
||||
Block: b.config.Block,
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read redis stream jobs: %w", err)
|
||||
}
|
||||
return b.deliveries(streams)
|
||||
}
|
||||
|
||||
func (b *Broker) Reclaim(ctx context.Context, start string) ([]Delivery, string, error) {
|
||||
messages, next, err := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
||||
Stream: b.config.Stream,
|
||||
Group: b.config.Group,
|
||||
Consumer: b.config.Consumer,
|
||||
MinIdle: b.config.ClaimMinIdle,
|
||||
Start: start,
|
||||
Count: b.config.ReadCount,
|
||||
}).Result()
|
||||
if err != nil {
|
||||
return nil, start, fmt.Errorf("reclaim redis stream jobs: %w", err)
|
||||
}
|
||||
|
||||
deliveries, err := b.deliveries([]redis.XStream{{Stream: b.config.Stream, Messages: messages}})
|
||||
if err != nil {
|
||||
return nil, start, err
|
||||
}
|
||||
return deliveries, next, nil
|
||||
}
|
||||
|
||||
func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) {
|
||||
var deliveries []Delivery
|
||||
for _, stream := range streams {
|
||||
for _, message := range stream.Messages {
|
||||
delivery := Delivery{ID: message.ID, broker: b}
|
||||
payload, err := messagePayload(message)
|
||||
if err != nil {
|
||||
delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err)
|
||||
deliveries = append(deliveries, delivery)
|
||||
continue
|
||||
}
|
||||
envelope, err := DecodeEnvelope(payload)
|
||||
if err != nil {
|
||||
delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err)
|
||||
deliveries = append(deliveries, delivery)
|
||||
continue
|
||||
}
|
||||
delivery.Envelope = envelope
|
||||
delivery.payload, err = EncodeEnvelope(envelope)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode validated redis stream message %s: %w", message.ID, err)
|
||||
}
|
||||
deliveries = append(deliveries, delivery)
|
||||
}
|
||||
}
|
||||
return deliveries, nil
|
||||
}
|
||||
|
||||
func messagePayload(message redis.XMessage) ([]byte, error) {
|
||||
value, ok := message.Values[payloadField]
|
||||
if !ok {
|
||||
return nil, errors.New("payload field is missing")
|
||||
}
|
||||
switch payload := value.(type) {
|
||||
case string:
|
||||
return []byte(payload), nil
|
||||
case []byte:
|
||||
return payload, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("payload has type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func (d Delivery) Ack(ctx context.Context) error {
|
||||
if d.broker == nil {
|
||||
return errors.New("delivery is detached from broker")
|
||||
}
|
||||
acked, err := d.broker.client.Eval(ctx, ackOwnedScript, []string{d.broker.config.Stream},
|
||||
d.broker.config.Group, d.ID, d.broker.config.Consumer,
|
||||
).Int64()
|
||||
if err != nil {
|
||||
return fmt.Errorf("ack redis stream job: %w", err)
|
||||
}
|
||||
if acked == 0 {
|
||||
return ErrDeliveryNotOwned
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const ackOwnedScript = `
|
||||
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
||||
if #pending == 0 or pending[1][2] ~= ARGV[3] then
|
||||
return 0
|
||||
end
|
||||
local acked = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
||||
if acked == 1 then
|
||||
redis.call('XDEL', KEYS[1], ARGV[2])
|
||||
end
|
||||
return acked
|
||||
`
|
||||
|
||||
func (d Delivery) ToFailed(ctx context.Context, failure FailedDelivery) error {
|
||||
if d.broker == nil {
|
||||
return errors.New("delivery is detached from broker")
|
||||
}
|
||||
if !failureCodePattern.MatchString(failure.Code) {
|
||||
return errors.New("failure code is invalid")
|
||||
}
|
||||
if failure.FailedAt.IsZero() {
|
||||
return errors.New("failure time is required")
|
||||
}
|
||||
|
||||
payload := d.payload
|
||||
if d.DecodeError == nil && len(payload) == 0 {
|
||||
var err error
|
||||
payload, err = EncodeEnvelope(d.Envelope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
moved, err := d.broker.client.Eval(ctx, moveToFailedScript,
|
||||
[]string{d.broker.config.FailedStream, d.broker.config.Stream},
|
||||
d.broker.config.Group, d.ID, d.broker.config.Consumer, string(payload), failure.Code,
|
||||
failure.FailedAt.UTC().Format(time.RFC3339Nano), strconv.Itoa(d.Envelope.Attempt),
|
||||
strconv.FormatInt(d.broker.config.FailedMaxLen, 10),
|
||||
).Int64()
|
||||
if err != nil {
|
||||
return fmt.Errorf("move redis stream job to failed stream: %w", err)
|
||||
}
|
||||
if moved == 0 {
|
||||
return ErrDeliveryNotOwned
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const moveToFailedScript = `
|
||||
local pending = redis.call('XPENDING', KEYS[2], ARGV[1], ARGV[2], ARGV[2], 1)
|
||||
if #pending == 0 or pending[1][2] ~= ARGV[3] then
|
||||
return 0
|
||||
end
|
||||
redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[8], '*',
|
||||
'payload', ARGV[4],
|
||||
'source_message_id', ARGV[2],
|
||||
'error_code', ARGV[5],
|
||||
'failed_at', ARGV[6],
|
||||
'attempt', ARGV[7])
|
||||
local acked = redis.call('XACK', KEYS[2], ARGV[1], ARGV[2])
|
||||
if acked == 1 then
|
||||
redis.call('XDEL', KEYS[2], ARGV[2])
|
||||
end
|
||||
return acked
|
||||
`
|
||||
@@ -0,0 +1,74 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBrokerDecodesDeliveries(t *testing.T) {
|
||||
payload, err := EncodeEnvelope(validEnvelope())
|
||||
require.NoError(t, err)
|
||||
|
||||
broker := &Broker{config: Config{Stream: "attachment-jobs"}}
|
||||
deliveries, err := broker.deliveries([]redis.XStream{{
|
||||
Stream: "attachment-jobs",
|
||||
Messages: []redis.XMessage{{
|
||||
ID: "1-0",
|
||||
Values: map[string]any{payloadField: string(payload)},
|
||||
}},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deliveries, 1)
|
||||
require.Equal(t, "1-0", deliveries[0].ID)
|
||||
require.Equal(t, validEnvelope(), deliveries[0].Envelope)
|
||||
}
|
||||
|
||||
func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) {
|
||||
broker := &Broker{config: Config{Stream: "attachment-jobs"}}
|
||||
deliveries, err := broker.deliveries([]redis.XStream{{
|
||||
Stream: "attachment-jobs",
|
||||
Messages: []redis.XMessage{{
|
||||
ID: "1-0",
|
||||
Values: map[string]any{payloadField: `{
|
||||
"schema_version":1,
|
||||
"job_id":"11111111-1111-4111-8111-111111111111",
|
||||
"type":"attachment.process.v1",
|
||||
"attachment_id":"22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version":1,
|
||||
"attempt":1,
|
||||
"created_at":"2026-09-01T12:00:00Z",
|
||||
"storage_key":"private/object"
|
||||
}`},
|
||||
}},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deliveries, 1)
|
||||
require.Error(t, deliveries[0].DecodeError)
|
||||
require.Empty(t, deliveries[0].payload)
|
||||
}
|
||||
|
||||
func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) {
|
||||
broker := &Broker{}
|
||||
_, err := broker.PublishConfirmed(t.Context(), []byte(`{"schema_version":2}`))
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestConfigValidation(t *testing.T) {
|
||||
config := Config{
|
||||
Stream: "attachment-jobs",
|
||||
Group: "attachment-workers",
|
||||
Consumer: "worker-1",
|
||||
FailedStream: "attachment-jobs-failed",
|
||||
FailedMaxLen: 1000,
|
||||
Block: time.Second,
|
||||
ClaimMinIdle: time.Minute,
|
||||
ReadCount: 10,
|
||||
}
|
||||
require.NoError(t, config.validate())
|
||||
|
||||
config.ClaimMinIdle = 0
|
||||
require.Error(t, config.validate())
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
SchemaVersion = 1
|
||||
JobType = "attachment.process.v1"
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidEnvelope = errors.New("invalid job envelope")
|
||||
traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
|
||||
uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||
)
|
||||
|
||||
type Envelope struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID uuid.UUID `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID uuid.UUID `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
func (e Envelope) Validate() error {
|
||||
switch {
|
||||
case e.SchemaVersion != SchemaVersion:
|
||||
return fmt.Errorf("%w: unsupported schema_version", errInvalidEnvelope)
|
||||
case e.Type != JobType:
|
||||
return fmt.Errorf("%w: unsupported type", errInvalidEnvelope)
|
||||
case e.JobID == uuid.Nil:
|
||||
return fmt.Errorf("%w: job_id is required", errInvalidEnvelope)
|
||||
case e.AttachmentID == uuid.Nil:
|
||||
return fmt.Errorf("%w: attachment_id is required", errInvalidEnvelope)
|
||||
case e.PipelineVersion < 1:
|
||||
return fmt.Errorf("%w: pipeline_version must be positive", errInvalidEnvelope)
|
||||
case e.Attempt < 1:
|
||||
return fmt.Errorf("%w: attempt must be positive", errInvalidEnvelope)
|
||||
case e.CreatedAt.IsZero():
|
||||
return fmt.Errorf("%w: created_at is required", errInvalidEnvelope)
|
||||
case e.Traceparent != "" && !traceparentPattern.MatchString(e.Traceparent):
|
||||
return fmt.Errorf("%w: invalid traceparent", errInvalidEnvelope)
|
||||
case utf8.RuneCountInString(e.Tracestate) > 512:
|
||||
return fmt.Errorf("%w: tracestate is too long", errInvalidEnvelope)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func EncodeEnvelope(envelope Envelope) ([]byte, error) {
|
||||
if err := envelope.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode job envelope: %w", err)
|
||||
}
|
||||
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func DecodeEnvelope(payload []byte) (Envelope, error) {
|
||||
var wire envelopeJSON
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&wire); err != nil {
|
||||
return Envelope{}, fmt.Errorf("%w: decode: %v", errInvalidEnvelope, err)
|
||||
}
|
||||
if err := ensureJSONEnd(decoder); err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
|
||||
jobID, err := parseUUID(wire.JobID, "job_id")
|
||||
if err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
attachmentID, err := parseUUID(wire.AttachmentID, "attachment_id")
|
||||
if err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
createdAt, err := time.Parse(time.RFC3339, wire.CreatedAt)
|
||||
if err != nil {
|
||||
return Envelope{}, fmt.Errorf("%w: invalid created_at", errInvalidEnvelope)
|
||||
}
|
||||
envelope := Envelope{
|
||||
SchemaVersion: wire.SchemaVersion,
|
||||
JobID: jobID,
|
||||
Type: wire.Type,
|
||||
AttachmentID: attachmentID,
|
||||
PipelineVersion: wire.PipelineVersion,
|
||||
Attempt: wire.Attempt,
|
||||
CreatedAt: createdAt,
|
||||
Traceparent: wire.Traceparent,
|
||||
Tracestate: wire.Tracestate,
|
||||
}
|
||||
if err := envelope.Validate(); err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
|
||||
return envelope, nil
|
||||
}
|
||||
|
||||
type envelopeJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
func parseUUID(value, field string) (uuid.UUID, error) {
|
||||
if !uuidPattern.MatchString(value) {
|
||||
return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field)
|
||||
}
|
||||
parsed, err := uuid.Parse(value)
|
||||
if err != nil {
|
||||
return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func ensureJSONEnd(decoder *json.Decoder) error {
|
||||
var value any
|
||||
if err := decoder.Decode(&value); !errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("%w: trailing JSON value", errInvalidEnvelope)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package redisstream
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEnvelopeRoundTrip(t *testing.T) {
|
||||
envelope := validEnvelope()
|
||||
payload, err := EncodeEnvelope(envelope)
|
||||
require.NoError(t, err)
|
||||
|
||||
decoded, err := DecodeEnvelope(payload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, envelope, decoded)
|
||||
}
|
||||
|
||||
func TestDecodeEnvelopeRejectsUnknownField(t *testing.T) {
|
||||
payload := []byte(`{
|
||||
"schema_version":1,
|
||||
"job_id":"11111111-1111-4111-8111-111111111111",
|
||||
"type":"attachment.process.v1",
|
||||
"attachment_id":"22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version":1,
|
||||
"attempt":1,
|
||||
"created_at":"2026-09-01T12:00:00Z",
|
||||
"storage_key":"must-not-enter-the-broker"
|
||||
}`)
|
||||
|
||||
_, err := DecodeEnvelope(payload)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestEnvelopeValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Envelope)
|
||||
}{
|
||||
{name: "schema", mutate: func(e *Envelope) { e.SchemaVersion = 2 }},
|
||||
{name: "type", mutate: func(e *Envelope) { e.Type = "attachment.delete.v1" }},
|
||||
{name: "job id", mutate: func(e *Envelope) { e.JobID = uuid.Nil }},
|
||||
{name: "attachment id", mutate: func(e *Envelope) { e.AttachmentID = uuid.Nil }},
|
||||
{name: "pipeline", mutate: func(e *Envelope) { e.PipelineVersion = 0 }},
|
||||
{name: "attempt", mutate: func(e *Envelope) { e.Attempt = 0 }},
|
||||
{name: "created at", mutate: func(e *Envelope) { e.CreatedAt = time.Time{} }},
|
||||
{name: "traceparent", mutate: func(e *Envelope) { e.Traceparent = "invalid" }},
|
||||
{name: "tracestate length", mutate: func(e *Envelope) { e.Tracestate = string(make([]byte, 513)) }},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
envelope := validEnvelope()
|
||||
tt.mutate(&envelope)
|
||||
require.Error(t, envelope.Validate())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validEnvelope() Envelope {
|
||||
return Envelope{
|
||||
SchemaVersion: SchemaVersion,
|
||||
JobID: uuid.MustParse("11111111-1111-4111-8111-111111111111"),
|
||||
Type: JobType,
|
||||
AttachmentID: uuid.MustParse("22222222-2222-4222-8222-222222222222"),
|
||||
PipelineVersion: 1,
|
||||
Attempt: 1,
|
||||
CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC),
|
||||
Traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
|
||||
Tracestate: "vendor=value",
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,7 @@ type ServerConfig struct {
|
||||
AttachmentMaxPixels int64
|
||||
AttachmentBodyMemory int64
|
||||
AttachmentDecodeMemory int64
|
||||
PipelineVersion int
|
||||
}
|
||||
|
||||
type MigrationConfig struct {
|
||||
@@ -207,6 +208,10 @@ func LoadServer() (ServerConfig, error) {
|
||||
if int64(attachmentDecodeMemory) < minimumDecodeMemory {
|
||||
return ServerConfig{}, fmt.Errorf("ATTACHMENT_DECODE_MEMORY_BYTES: must be at least %d", minimumDecodeMemory)
|
||||
}
|
||||
pipelineVersion, err := integer(values, "PIPELINE_VERSION", 1, 1000)
|
||||
if err != nil {
|
||||
return ServerConfig{}, err
|
||||
}
|
||||
|
||||
return ServerConfig{
|
||||
Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL,
|
||||
@@ -226,6 +231,7 @@ func LoadServer() (ServerConfig, error) {
|
||||
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||
AttachmentBodyMemory: int64(attachmentBodyMemory),
|
||||
AttachmentDecodeMemory: int64(attachmentDecodeMemory),
|
||||
PipelineVersion: pipelineVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
type DispatcherConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
BrokerURL string
|
||||
Stream string
|
||||
ConsumerGroup string
|
||||
FailedStream string
|
||||
FailedStreamMaxLength int64
|
||||
BatchSize int
|
||||
LeaseDuration time.Duration
|
||||
PollInterval time.Duration
|
||||
RecoveryInterval time.Duration
|
||||
RetryDelay time.Duration
|
||||
MetricsAddress string
|
||||
}
|
||||
|
||||
type WorkerConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKey string
|
||||
S3SecretKey string
|
||||
S3UseTLS bool
|
||||
S3Timeout time.Duration
|
||||
BrokerURL string
|
||||
Stream string
|
||||
ConsumerGroup string
|
||||
FailedStream string
|
||||
FailedStreamMaxLength int64
|
||||
Concurrency int
|
||||
Prefetch int
|
||||
LeaseDuration time.Duration
|
||||
ProcessingTimeout time.Duration
|
||||
ReadBlock time.Duration
|
||||
ReclaimInterval time.Duration
|
||||
MaxAttempts int
|
||||
RetryBase time.Duration
|
||||
RetryMax time.Duration
|
||||
AttachmentMaxBytes int64
|
||||
AttachmentMaxPixels int64
|
||||
MemoryBytes int64
|
||||
MetricsAddress string
|
||||
}
|
||||
|
||||
type JobAdminConfig struct {
|
||||
DatabaseURL string
|
||||
DBMaxConns int32
|
||||
DBMinConns int32
|
||||
DBMaxConnectionAge time.Duration
|
||||
DBMaxConnectionIdle time.Duration
|
||||
DBHealthCheckPeriod time.Duration
|
||||
DependencyTimeout time.Duration
|
||||
}
|
||||
|
||||
type brokerConfig struct {
|
||||
url string
|
||||
stream string
|
||||
consumerGroup string
|
||||
failedStream string
|
||||
failedStreamMaxLength int
|
||||
}
|
||||
|
||||
func LoadJobAdmin() (JobAdminConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return JobAdminConfig{}, err
|
||||
}
|
||||
return JobAdminConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadDispatcher() (DispatcherConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
broker, err := loadBrokerConfig(values)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
batchSize, err := integer(values, "OUTBOX_BATCH_SIZE", 1, 1000)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
leaseDuration, err := seconds(values, "OUTBOX_LEASE_SECONDS", 1, 600)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
pollInterval, err := seconds(values, "OUTBOX_POLL_SECONDS", 1, 60)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
recoveryInterval, err := seconds(values, "OUTBOX_RECOVERY_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
retryDelay, err := seconds(values, "OUTBOX_RETRY_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
metricsAddress, err := metricsAddress(values)
|
||||
if err != nil {
|
||||
return DispatcherConfig{}, err
|
||||
}
|
||||
return DispatcherConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup,
|
||||
FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
||||
BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval,
|
||||
RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, MetricsAddress: metricsAddress,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func LoadWorker() (WorkerConfig, error) {
|
||||
values := viper.New()
|
||||
values.AutomaticEnv()
|
||||
database, err := loadDatabaseConfig(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
storage, err := loadObjectStorageConfig(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
broker, err := loadBrokerConfig(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
concurrency, err := integer(values, "WORKER_CONCURRENCY", 1, 64)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
prefetch, err := integer(values, "WORKER_PREFETCH", concurrency, 1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
leaseDuration, err := seconds(values, "WORKER_LEASE_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
processingTimeout, err := seconds(values, "WORKER_PROCESSING_TIMEOUT_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if processingTimeout >= leaseDuration {
|
||||
return WorkerConfig{}, errors.New("WORKER_PROCESSING_TIMEOUT_SECONDS must be less than WORKER_LEASE_SECONDS")
|
||||
}
|
||||
readBlock, err := seconds(values, "WORKER_READ_BLOCK_SECONDS", 1, 60)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
reclaimInterval, err := seconds(values, "WORKER_RECLAIM_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
maxAttempts, err := integer(values, "WORKER_MAX_ATTEMPTS", 1, 100)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
retryBase, err := seconds(values, "WORKER_RETRY_BASE_SECONDS", 1, 3600)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
retryMax, err := seconds(values, "WORKER_RETRY_MAX_SECONDS", 1, 86400)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
if retryMax < retryBase {
|
||||
return WorkerConfig{}, errors.New("WORKER_RETRY_MAX_SECONDS must be at least WORKER_RETRY_BASE_SECONDS")
|
||||
}
|
||||
attachmentMaxBytes, err := integer(values, "ATTACHMENT_MAX_BYTES", 1, 10*1024*1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
attachmentMaxPixels, err := integer(values, "ATTACHMENT_MAX_PIXELS", 1, 40_000_000)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
memoryBytes, err := integer(values, "WORKER_MEMORY_BYTES", attachmentMaxBytes, 2*1024*1024*1024)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
metricsAddress, err := metricsAddress(values)
|
||||
if err != nil {
|
||||
return WorkerConfig{}, err
|
||||
}
|
||||
return WorkerConfig{
|
||||
DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns),
|
||||
DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle,
|
||||
DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout,
|
||||
S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket,
|
||||
S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS,
|
||||
S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream,
|
||||
ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream,
|
||||
FailedStreamMaxLength: int64(broker.failedStreamMaxLength),
|
||||
Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration,
|
||||
ProcessingTimeout: processingTimeout, ReadBlock: readBlock,
|
||||
ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax,
|
||||
AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels),
|
||||
MemoryBytes: int64(memoryBytes),
|
||||
MetricsAddress: metricsAddress,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func metricsAddress(values *viper.Viper) (string, error) {
|
||||
host, err := requiredString(values, "PIPELINE_METRICS_HOST")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
port, err := requiredPort(values, "PIPELINE_METRICS_PORT")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return net.JoinHostPort(host, port), nil
|
||||
}
|
||||
|
||||
func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) {
|
||||
url, err := requiredString(values, "BROKER_URL")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
stream, err := requiredString(values, "BROKER_STREAM")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
consumerGroup, err := requiredString(values, "BROKER_CONSUMER_GROUP")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
failedStream, err := requiredString(values, "BROKER_FAILED_STREAM")
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
failedStreamMaxLength, err := integer(values, "BROKER_FAILED_MAX_LENGTH", 100, 10_000_000)
|
||||
if err != nil {
|
||||
return brokerConfig{}, err
|
||||
}
|
||||
return brokerConfig{
|
||||
url: url, stream: stream, consumerGroup: consumerGroup,
|
||||
failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
type Metrics struct {
|
||||
workerInFlight prometheus.Gauge
|
||||
workerJobs *prometheus.CounterVec
|
||||
workerDuration *prometheus.HistogramVec
|
||||
workerRetries prometheus.Counter
|
||||
workerReclaimed prometheus.Counter
|
||||
outboxPublished prometheus.Counter
|
||||
outboxPublishFailure prometheus.Counter
|
||||
outboxRecovered prometheus.Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func NewMetrics() *Metrics {
|
||||
registry := prometheus.NewRegistry()
|
||||
metrics := &Metrics{
|
||||
workerInFlight: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_in_flight",
|
||||
Help: "Attachment jobs currently processed by this worker.",
|
||||
}),
|
||||
workerJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "jobs_total",
|
||||
Help: "Completed attachment job deliveries by result.",
|
||||
}, []string{"result"}),
|
||||
workerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "job_duration_seconds",
|
||||
Help: "Attachment job delivery processing duration.", Buckets: prometheus.DefBuckets,
|
||||
}, []string{"result"}),
|
||||
workerRetries: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "retries_total",
|
||||
Help: "Attachment jobs scheduled for another attempt.",
|
||||
}),
|
||||
workerReclaimed: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "attachment_worker", Name: "reclaimed_deliveries_total",
|
||||
Help: "Redis Stream deliveries reclaimed after their idle timeout.",
|
||||
}),
|
||||
outboxPublished: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "published_total",
|
||||
Help: "Outbox events confirmed by Redis Streams.",
|
||||
}),
|
||||
outboxPublishFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "publish_failures_total",
|
||||
Help: "Outbox events whose Redis Streams publish failed.",
|
||||
}),
|
||||
outboxRecovered: prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "messenger", Subsystem: "outbox", Name: "recovered_leases_total",
|
||||
Help: "Expired outbox leases returned for publication.",
|
||||
}),
|
||||
}
|
||||
registry.MustRegister(
|
||||
metrics.workerInFlight, metrics.workerJobs, metrics.workerDuration,
|
||||
metrics.workerRetries, metrics.workerReclaimed, metrics.outboxPublished,
|
||||
metrics.outboxPublishFailure, metrics.outboxRecovered,
|
||||
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
metrics.handler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
|
||||
return metrics
|
||||
}
|
||||
|
||||
func (m *Metrics) Handler() http.Handler {
|
||||
return m.handler
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveStarted() {
|
||||
m.workerInFlight.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveFinished(result string, duration time.Duration) {
|
||||
m.workerInFlight.Dec()
|
||||
m.workerJobs.WithLabelValues(result).Inc()
|
||||
m.workerDuration.WithLabelValues(result).Observe(duration.Seconds())
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRetry() {
|
||||
m.workerRetries.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveReclaimed(count int) {
|
||||
m.workerReclaimed.Add(float64(count))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublished() {
|
||||
m.outboxPublished.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObservePublishFailure() {
|
||||
m.outboxPublishFailure.Inc()
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRecovered(count int) {
|
||||
m.outboxRecovered.Add(float64(count))
|
||||
}
|
||||
@@ -25,10 +25,10 @@ import (
|
||||
type Repository interface {
|
||||
ChatForUser(context.Context, string, string) (model.Chat, error)
|
||||
CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error)
|
||||
MarkAttachmentReady(context.Context, string, string) (model.Attachment, error)
|
||||
QueueAttachmentWithJobAndOutbox(context.Context, string, string, model.AttachmentJob, string, string) (model.Attachment, error)
|
||||
DeletePendingAttachment(context.Context, string, string) error
|
||||
AttachmentByStorage(context.Context, string, string) (model.Attachment, error)
|
||||
AttachmentForUser(context.Context, string, string) (model.Attachment, error)
|
||||
AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
@@ -43,6 +43,7 @@ type Config struct {
|
||||
BodyMemoryBytes int64
|
||||
DecodeMemoryBytes int64
|
||||
CleanupTimeout time.Duration
|
||||
PipelineVersion int
|
||||
}
|
||||
|
||||
type Upload struct {
|
||||
@@ -52,10 +53,13 @@ type Upload struct {
|
||||
ContentType string
|
||||
ContentLength int64
|
||||
Body io.Reader
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
}
|
||||
|
||||
type Content struct {
|
||||
Attachment model.Attachment
|
||||
Variant model.Variant
|
||||
Body io.ReadCloser
|
||||
}
|
||||
|
||||
@@ -69,7 +73,7 @@ type Service struct {
|
||||
}
|
||||
|
||||
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 || config.PipelineVersion < 1 {
|
||||
return nil, errors.New("invalid attachment service configuration")
|
||||
}
|
||||
if config.BodyMemoryBytes < config.MaxBytes {
|
||||
@@ -96,58 +100,38 @@ func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, e
|
||||
defer release()
|
||||
|
||||
attachmentID := uuid.NewString()
|
||||
objectKey := "attachments/" + uuid.NewString() + "/original"
|
||||
stagingKey := "staging/" + attachmentID + "/" + uuid.NewString()
|
||||
digest := sha256.Sum256(data)
|
||||
pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{
|
||||
ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename,
|
||||
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
|
||||
ObjectKey: objectKey, Status: model.AttachmentPending,
|
||||
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending,
|
||||
PipelineVersion: s.config.PipelineVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
if err := s.objects.Put(ctx, objectKey, contentType, data); err != nil {
|
||||
s.deletePending(pending.ID, objectKey)
|
||||
if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil {
|
||||
s.logger.Error("attachment staging write failed", "attachment_id", pending.ID, "error", err)
|
||||
s.deletePending(pending.ID, stagingKey)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
attachment, err := s.repository.MarkAttachmentReady(ctx, pending.ID, objectKey)
|
||||
if err == nil {
|
||||
return attachment, nil
|
||||
createdAt := time.Now().UTC()
|
||||
queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion,
|
||||
Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt,
|
||||
}, input.Traceparent, input.Tracestate)
|
||||
if err != nil {
|
||||
s.logger.Error("attachment queue transaction failed", "attachment_id", pending.ID, "error", err)
|
||||
return model.Attachment{}, mapDependencyError(err)
|
||||
}
|
||||
return s.resolveReadyOutcome(pending.ID, objectKey)
|
||||
return queued, nil
|
||||
}
|
||||
|
||||
func (s *Service) resolveReadyOutcome(attachmentID, objectKey string) (model.Attachment, error) {
|
||||
func (s *Service) deletePending(attachmentID, stagingKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
stored, err := s.repository.AttachmentByStorage(cleanupCtx, attachmentID, objectKey)
|
||||
if err == nil && stored.Status == model.AttachmentReady {
|
||||
return stored, nil
|
||||
}
|
||||
if err == nil && stored.Status == model.AttachmentPending {
|
||||
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||
s.logger.Error("attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
if deleteErr := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); deleteErr != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", deleteErr)
|
||||
}
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
if errors.Is(err, model.ErrNotFound) {
|
||||
if deleteErr := s.objects.Delete(cleanupCtx, objectKey); deleteErr != nil {
|
||||
s.logger.Error("orphan attachment compensation failed", "object_key", objectKey, "error", deleteErr)
|
||||
}
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
s.logger.Error("attachment outcome is unknown", "attachment_id", attachmentID, "error", err)
|
||||
return model.Attachment{}, model.ErrUnavailable
|
||||
}
|
||||
|
||||
func (s *Service) deletePending(attachmentID, objectKey string) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), s.config.CleanupTimeout)
|
||||
defer cancel()
|
||||
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, objectKey); err != nil {
|
||||
if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil {
|
||||
s.logger.Error("pending attachment cleanup failed", "attachment_id", attachmentID, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -160,16 +144,16 @@ func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (mo
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *Service) Content(ctx context.Context, attachmentID, userID string) (Content, error) {
|
||||
attachment, err := s.Metadata(ctx, attachmentID, userID)
|
||||
func (s *Service) Content(ctx context.Context, attachmentID, userID, variantName string) (Content, error) {
|
||||
attachment, variant, err := s.repository.AttachmentVariantForUser(ctx, attachmentID, userID, variantName)
|
||||
if err != nil {
|
||||
return Content{}, err
|
||||
return Content{}, mapDependencyError(err)
|
||||
}
|
||||
body, err := s.objects.Get(ctx, attachment.ObjectKey)
|
||||
body, err := s.objects.Get(ctx, variant.ObjectKey)
|
||||
if err != nil {
|
||||
return Content{}, model.ErrUnavailable
|
||||
}
|
||||
return Content{Attachment: attachment, Body: body}, nil
|
||||
return Content{Attachment: attachment, Variant: variant, Body: body}, nil
|
||||
}
|
||||
|
||||
func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
|
||||
|
||||
@@ -27,8 +27,8 @@ func TestUploadLifecycle(t *testing.T) {
|
||||
|
||||
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, model.AttachmentReady, result.Status)
|
||||
require.Equal(t, []string{"pending", "put", "ready"}, events)
|
||||
require.Equal(t, model.AttachmentQueued, result.Status)
|
||||
require.Equal(t, []string{"pending", "put", "queued"}, events)
|
||||
}
|
||||
|
||||
func TestUploadValidation(t *testing.T) {
|
||||
@@ -86,7 +86,7 @@ func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
|
||||
Config{
|
||||
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
|
||||
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
|
||||
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, PipelineVersion: 1,
|
||||
},
|
||||
)
|
||||
require.Error(t, err)
|
||||
@@ -103,16 +103,16 @@ func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
|
||||
require.True(t, repository.pendingDeleted)
|
||||
}
|
||||
|
||||
func TestUploadReturnsCommittedMetadataAfterAmbiguousReadyUpdate(t *testing.T) {
|
||||
func TestUploadKeepsStagingAfterQueueFailure(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
repository := &repositoryStub{markErr: errors.New("connection lost"), lookupReady: true}
|
||||
repository := &repositoryStub{queueErr: errors.New("connection lost")}
|
||||
objects := &objectStub{}
|
||||
service := newTestService(t, repository, objects, 1024, 4)
|
||||
|
||||
result, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, model.AttachmentReady, result.Status)
|
||||
_, err := service.Upload(context.Background(), upload(data, "image/png"))
|
||||
require.ErrorIs(t, err, model.ErrUnavailable)
|
||||
require.False(t, objects.deleted)
|
||||
require.False(t, repository.pendingDeleted)
|
||||
}
|
||||
|
||||
func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service {
|
||||
@@ -121,7 +121,7 @@ func newTestService(t *testing.T, repository Repository, objects ObjectStore, ma
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects,
|
||||
Config{
|
||||
MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes,
|
||||
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second,
|
||||
DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, PipelineVersion: 1,
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
@@ -137,8 +137,7 @@ func upload(data []byte, contentType string) Upload {
|
||||
|
||||
type repositoryStub struct {
|
||||
events *[]string
|
||||
markErr error
|
||||
lookupReady bool
|
||||
queueErr error
|
||||
pendingDeleted bool
|
||||
pending model.Attachment
|
||||
}
|
||||
@@ -154,12 +153,19 @@ func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment m
|
||||
return attachment, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) MarkAttachmentReady(context.Context, string, string) (model.Attachment, error) {
|
||||
if s.markErr != nil {
|
||||
return model.Attachment{}, s.markErr
|
||||
func (s *repositoryStub) QueueAttachmentWithJobAndOutbox(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
model.AttachmentJob,
|
||||
string,
|
||||
string,
|
||||
) (model.Attachment, error) {
|
||||
if s.queueErr != nil {
|
||||
return model.Attachment{}, s.queueErr
|
||||
}
|
||||
s.pending.Status = model.AttachmentReady
|
||||
s.record("ready")
|
||||
s.pending.Status = model.AttachmentQueued
|
||||
s.record("queued")
|
||||
return s.pending, nil
|
||||
}
|
||||
|
||||
@@ -168,18 +174,14 @@ func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentByStorage(context.Context, string, string) (model.Attachment, error) {
|
||||
if s.lookupReady {
|
||||
s.pending.Status = model.AttachmentReady
|
||||
return s.pending, nil
|
||||
}
|
||||
return model.Attachment{}, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) {
|
||||
return s.pending, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) {
|
||||
return s.pending, model.Variant{}, nil
|
||||
}
|
||||
|
||||
func (s *repositoryStub) record(event string) {
|
||||
if s.events != nil {
|
||||
*s.events = append(*s.events, event)
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
)
|
||||
|
||||
var errInvalidEXIF = errors.New("invalid EXIF metadata")
|
||||
|
||||
func jpegOrientation(data []byte) (int, error) {
|
||||
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for offset := 2; offset < len(data); {
|
||||
if data[offset] != 0xff {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for offset < len(data) && data[offset] == 0xff {
|
||||
offset++
|
||||
}
|
||||
if offset >= len(data) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
marker := data[offset]
|
||||
offset++
|
||||
if marker == 0xd9 || marker == 0xda {
|
||||
return 1, nil
|
||||
}
|
||||
if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 {
|
||||
continue
|
||||
}
|
||||
if offset+2 > len(data) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
length := int(binary.BigEndian.Uint16(data[offset : offset+2]))
|
||||
if length < 2 || offset+length > len(data) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
payload := data[offset+2 : offset+length]
|
||||
offset += length
|
||||
if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" {
|
||||
continue
|
||||
}
|
||||
return tiffOrientation(payload[6:])
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func tiffOrientation(data []byte) (int, error) {
|
||||
if len(data) < 8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
var order binary.ByteOrder
|
||||
switch string(data[:2]) {
|
||||
case "II":
|
||||
order = binary.LittleEndian
|
||||
case "MM":
|
||||
order = binary.BigEndian
|
||||
default:
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
if order.Uint16(data[2:4]) != 42 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
ifdOffset := uint64(order.Uint32(data[4:8]))
|
||||
if ifdOffset > uint64(len(data)-2) {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
count := uint64(order.Uint16(data[ifdOffset : ifdOffset+2]))
|
||||
entriesStart := ifdOffset + 2
|
||||
if count > (uint64(len(data))-entriesStart)/12 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
for index := uint64(0); index < count; index++ {
|
||||
entry := data[entriesStart+index*12 : entriesStart+(index+1)*12]
|
||||
if order.Uint16(entry[:2]) != 0x0112 {
|
||||
continue
|
||||
}
|
||||
if order.Uint16(entry[2:4]) != 3 || order.Uint32(entry[4:8]) != 1 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
orientation := int(order.Uint16(entry[8:10]))
|
||||
if orientation < 1 || orientation > 8 {
|
||||
return 0, errInvalidEXIF
|
||||
}
|
||||
return orientation, nil
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func applyOrientation(source image.Image, orientation int) *image.NRGBA {
|
||||
bounds := source.Bounds()
|
||||
width, height := bounds.Dx(), bounds.Dy()
|
||||
outputWidth, outputHeight := width, height
|
||||
if orientation >= 5 {
|
||||
outputWidth, outputHeight = height, width
|
||||
}
|
||||
output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight))
|
||||
for y := 0; y < outputHeight; y++ {
|
||||
for x := 0; x < outputWidth; x++ {
|
||||
sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation)
|
||||
output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY))
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func orientedSourcePoint(x, y, width, height, orientation int) (int, int) {
|
||||
switch orientation {
|
||||
case 2:
|
||||
return width - 1 - x, y
|
||||
case 3:
|
||||
return width - 1 - x, height - 1 - y
|
||||
case 4:
|
||||
return x, height - 1 - y
|
||||
case 5:
|
||||
return y, x
|
||||
case 6:
|
||||
return y, height - 1 - x
|
||||
case 7:
|
||||
return width - 1 - y, height - 1 - x
|
||||
case 8:
|
||||
return width - 1 - y, x
|
||||
default:
|
||||
return x, y
|
||||
}
|
||||
}
|
||||
|
||||
func flattenWhite(source image.Image) *image.NRGBA {
|
||||
bounds := source.Bounds()
|
||||
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
|
||||
for y := 0; y < bounds.Dy(); y++ {
|
||||
for x := 0; x < bounds.Dx(); x++ {
|
||||
r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA()
|
||||
output.SetNRGBA(x, y, color.NRGBA{
|
||||
R: uint8((r + 0xffff - a) / 0x101),
|
||||
G: uint8((g + 0xffff - a) / 0x101),
|
||||
B: uint8((b + 0xffff - a) / 0x101),
|
||||
A: 0xff,
|
||||
})
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
"github.com/deepteams/webp"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultJPEGQuality is used for thumbnail and crop variants.
|
||||
DefaultJPEGQuality = 85
|
||||
// DefaultWebPQuality keeps the optimized variant visually close to the source while reducing its size.
|
||||
DefaultWebPQuality = 82
|
||||
// DefaultWebPMethod selects the encoder's balanced quality and CPU mode.
|
||||
DefaultWebPMethod = 4
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidConfig = errors.New("invalid image transform configuration")
|
||||
ErrUnsupportedMedia = errors.New("unsupported image media type")
|
||||
ErrPayloadTooLarge = errors.New("image exceeds configured limits")
|
||||
ErrInvalidImage = errors.New("invalid image data")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
JPEGQuality int
|
||||
WebPQuality float32
|
||||
WebPMethod int
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Bytes []byte
|
||||
ContentType string
|
||||
Width int
|
||||
Height int
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Original Variant
|
||||
Thumbnail Variant
|
||||
Crop Variant
|
||||
Optimized Variant
|
||||
Orientation int
|
||||
}
|
||||
|
||||
type Processor struct {
|
||||
config Config
|
||||
}
|
||||
|
||||
func DefaultConfig(maxBytes, maxPixels int64) Config {
|
||||
return Config{
|
||||
MaxBytes: maxBytes, MaxPixels: maxPixels,
|
||||
JPEGQuality: DefaultJPEGQuality, WebPQuality: DefaultWebPQuality, WebPMethod: DefaultWebPMethod,
|
||||
}
|
||||
}
|
||||
|
||||
func NewProcessor(config Config) (*Processor, error) {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.JPEGQuality < 1 || config.JPEGQuality > 100 ||
|
||||
config.WebPQuality < 1 || config.WebPQuality > 100 || config.WebPMethod < 0 || config.WebPMethod > 6 {
|
||||
return nil, ErrInvalidConfig
|
||||
}
|
||||
return &Processor{config: config}, nil
|
||||
}
|
||||
|
||||
func (p *Processor) Process(ctx context.Context, data []byte, declaredContentType string) (Result, error) {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
img, format, width, height, orientation, err := p.decode(data, declaredContentType)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
oriented := applyOrientation(img, orientation)
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
opaque := flattenWhite(oriented)
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result := Result{
|
||||
Original: variant(data, format, width, height),
|
||||
Orientation: orientation,
|
||||
}
|
||||
|
||||
thumbnail := fit(opaque, 256, 256, false)
|
||||
result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
crop := centeredSquare(opaque)
|
||||
crop = resize(crop, 128, 128)
|
||||
result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
optimized := fit(opaque, 1280, 1280, false)
|
||||
result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *Processor) decode(data []byte, declaredContentType string) (image.Image, string, int, int, int, error) {
|
||||
if declaredContentType != "image/jpeg" && declaredContentType != "image/png" {
|
||||
return nil, "", 0, 0, 0, ErrUnsupportedMedia
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
if int64(len(data)) > p.config.MaxBytes {
|
||||
return nil, "", 0, 0, 0, ErrPayloadTooLarge
|
||||
}
|
||||
if http.DetectContentType(data) != declaredContentType {
|
||||
return nil, "", 0, 0, 0, ErrUnsupportedMedia
|
||||
}
|
||||
config, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
if int64(config.Width) > p.config.MaxPixels/int64(config.Height) {
|
||||
return nil, "", 0, 0, 0, ErrPayloadTooLarge
|
||||
}
|
||||
img, decodedFormat, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil || decodedFormat != format {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
orientation := 1
|
||||
if format == "jpeg" {
|
||||
orientation, err = jpegOrientation(data)
|
||||
if err != nil {
|
||||
return nil, "", 0, 0, 0, ErrInvalidImage
|
||||
}
|
||||
}
|
||||
return img, declaredContentType, config.Width, config.Height, orientation, nil
|
||||
}
|
||||
|
||||
func contextError(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func contentType(format string) string {
|
||||
switch format {
|
||||
case "jpeg":
|
||||
return "image/jpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func encodeJPEG(img image.Image, quality int) (Variant, error) {
|
||||
var output bytes.Buffer
|
||||
if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: quality}); err != nil {
|
||||
return Variant{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
return variant(output.Bytes(), "image/jpeg", bounds.Dx(), bounds.Dy()), nil
|
||||
}
|
||||
|
||||
func encodeWebP(img image.Image, quality float32, method int) (Variant, error) {
|
||||
var output bytes.Buffer
|
||||
options := &webp.EncoderOptions{Quality: quality, Method: method}
|
||||
if err := webp.Encode(&output, img, options); err != nil {
|
||||
return Variant{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
return variant(output.Bytes(), "image/webp", bounds.Dx(), bounds.Dy()), nil
|
||||
}
|
||||
|
||||
func variant(data []byte, contentType string, width, height int) Variant {
|
||||
digest := sha256.Sum256(data)
|
||||
return Variant{
|
||||
Bytes: data, ContentType: contentType, Width: width, Height: height,
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
}
|
||||
}
|
||||
|
||||
// EstimateMemoryWeight returns a conservative reservation for the input, decoded source,
|
||||
// orientation and alpha copies, variant working buffer, and encoded variant buffers.
|
||||
func EstimateMemoryWeight(inputBytes int64, width, height int) (int64, error) {
|
||||
if inputBytes < 0 || width < 1 || height < 1 || int64(width) > math.MaxInt64/int64(height) {
|
||||
return 0, ErrInvalidImage
|
||||
}
|
||||
pixels := int64(width) * int64(height)
|
||||
optimizedPixels := fittedPixels(width, height, 1280, 1280)
|
||||
thumbnailPixels := fittedPixels(width, height, 256, 256)
|
||||
const cropPixels int64 = 128 * 128
|
||||
if pixels > (math.MaxInt64-inputBytes)/16 {
|
||||
return 0, ErrPayloadTooLarge
|
||||
}
|
||||
weight := inputBytes + pixels*16
|
||||
variants := (optimizedPixels + thumbnailPixels + cropPixels) * 4
|
||||
if variants > math.MaxInt64-weight {
|
||||
return 0, ErrPayloadTooLarge
|
||||
}
|
||||
return weight + variants, nil
|
||||
}
|
||||
|
||||
func fittedPixels(width, height, maxWidth, maxHeight int) int64 {
|
||||
newWidth, newHeight := fitDimensions(width, height, maxWidth, maxHeight, false)
|
||||
return int64(newWidth) * int64(newHeight)
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"testing"
|
||||
|
||||
"github.com/deepteams/webp"
|
||||
)
|
||||
|
||||
func TestProcessProducesExpectedVariants(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 320, 200))
|
||||
fillGradient(source)
|
||||
data := encodePNG(t, source)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 320*200)
|
||||
|
||||
result, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("process image: %v", err)
|
||||
}
|
||||
|
||||
assertVariant(t, result.Original, "image/png", 320, 200)
|
||||
assertVariant(t, result.Thumbnail, "image/jpeg", 256, 160)
|
||||
assertVariant(t, result.Crop, "image/jpeg", 128, 128)
|
||||
assertVariant(t, result.Optimized, "image/webp", 320, 200)
|
||||
if !bytes.Equal(result.Original.Bytes, data) {
|
||||
t.Fatal("original bytes changed")
|
||||
}
|
||||
digest := sha256.Sum256(data)
|
||||
if result.Original.SHA256 != hex.EncodeToString(digest[:]) {
|
||||
t.Fatalf("unexpected original digest: %s", result.Original.SHA256)
|
||||
}
|
||||
if _, err := webp.Decode(bytes.NewReader(result.Optimized.Bytes)); err != nil {
|
||||
t.Fatalf("decode optimized WebP: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessDoesNotUpscaleFitVariants(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 64, 32))
|
||||
fillGradient(source)
|
||||
data := encodePNG(t, source)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 64*32)
|
||||
|
||||
result, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("process image: %v", err)
|
||||
}
|
||||
|
||||
assertVariant(t, result.Thumbnail, "image/jpeg", 64, 32)
|
||||
assertVariant(t, result.Optimized, "image/webp", 64, 32)
|
||||
assertVariant(t, result.Crop, "image/jpeg", 128, 128)
|
||||
}
|
||||
|
||||
func TestFlattenWhiteCompositesAlpha(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 2, 1))
|
||||
source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0})
|
||||
source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128})
|
||||
|
||||
result := flattenWhite(source)
|
||||
if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) {
|
||||
t.Fatalf("transparent pixel: got %#v", got)
|
||||
}
|
||||
if got := result.NRGBAAt(1, 0); got != (color.NRGBA{R: 177, G: 152, B: 127, A: 255}) {
|
||||
t.Fatalf("semi-transparent pixel: got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessAppliesEXIFOrientation(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 3, 2))
|
||||
fillGradient(source)
|
||||
data := injectOrientation(t, encodeJPEGForTest(t, source), 6)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 3*2)
|
||||
|
||||
result, err := processor.Process(context.Background(), data, "image/jpeg")
|
||||
if err != nil {
|
||||
t.Fatalf("process image: %v", err)
|
||||
}
|
||||
|
||||
if result.Orientation != 6 {
|
||||
t.Fatalf("orientation: got %d, want 6", result.Orientation)
|
||||
}
|
||||
assertVariant(t, result.Original, "image/jpeg", 3, 2)
|
||||
assertVariant(t, result.Thumbnail, "image/jpeg", 2, 3)
|
||||
assertVariant(t, result.Optimized, "image/webp", 2, 3)
|
||||
}
|
||||
|
||||
func TestProcessRejectsInvalidInputs(t *testing.T) {
|
||||
valid := encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 11, 10)))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
contentType string
|
||||
maxPixels int64
|
||||
want error
|
||||
}{
|
||||
{name: "MIME mismatch", data: valid, contentType: "image/jpeg", maxPixels: 200, want: ErrUnsupportedMedia},
|
||||
{name: "unsupported MIME", data: valid, contentType: "image/gif", maxPixels: 200, want: ErrUnsupportedMedia},
|
||||
{name: "pixel limit", data: valid, contentType: "image/png", maxPixels: 100, want: ErrPayloadTooLarge},
|
||||
{name: "corrupted", data: valid[:len(valid)-12], contentType: "image/png", maxPixels: 200, want: ErrInvalidImage},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
processor := newTestProcessor(t, int64(len(valid))+1, test.maxPixels)
|
||||
_, err := processor.Process(context.Background(), test.data, test.contentType)
|
||||
if !errors.Is(err, test.want) {
|
||||
t.Fatalf("error: got %v, want %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessIsDeterministic(t *testing.T) {
|
||||
source := image.NewNRGBA(image.Rect(0, 0, 48, 31))
|
||||
fillGradient(source)
|
||||
data := encodePNG(t, source)
|
||||
processor := newTestProcessor(t, int64(len(data))+1, 48*31)
|
||||
|
||||
first, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("first process: %v", err)
|
||||
}
|
||||
second, err := processor.Process(context.Background(), data, "image/png")
|
||||
if err != nil {
|
||||
t.Fatalf("second process: %v", err)
|
||||
}
|
||||
|
||||
for name, variants := range map[string][2]Variant{
|
||||
"thumbnail": {first.Thumbnail, second.Thumbnail},
|
||||
"crop": {first.Crop, second.Crop},
|
||||
"optimized": {first.Optimized, second.Optimized},
|
||||
} {
|
||||
if !bytes.Equal(variants[0].Bytes, variants[1].Bytes) {
|
||||
t.Fatalf("%s bytes differ", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessHonorsCanceledContext(t *testing.T) {
|
||||
processor := newTestProcessor(t, 1024, 100)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := processor.Process(ctx, []byte("unused"), "image/png")
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error: got %v, want %v", err, context.Canceled)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEstimateMemoryWeight(t *testing.T) {
|
||||
weight, err := EstimateMemoryWeight(1000, 320, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("estimate memory: %v", err)
|
||||
}
|
||||
want := int64(1000 + 320*200*16 + (320*200+256*160+128*128)*4)
|
||||
if weight != want {
|
||||
t.Fatalf("weight: got %d, want %d", weight, want)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestProcessor(t *testing.T, maxBytes, maxPixels int64) *Processor {
|
||||
t.Helper()
|
||||
processor, err := NewProcessor(DefaultConfig(maxBytes, maxPixels))
|
||||
if err != nil {
|
||||
t.Fatalf("new processor: %v", err)
|
||||
}
|
||||
return processor
|
||||
}
|
||||
|
||||
func assertVariant(t *testing.T, variant Variant, contentType string, width, height int) {
|
||||
t.Helper()
|
||||
if variant.ContentType != contentType || variant.Width != width || variant.Height != height {
|
||||
t.Fatalf("variant: got type=%s dimensions=%dx%d, want type=%s dimensions=%dx%d",
|
||||
variant.ContentType, variant.Width, variant.Height, contentType, width, height)
|
||||
}
|
||||
if len(variant.Bytes) == 0 || variant.SHA256 == "" {
|
||||
t.Fatal("variant has no bytes or digest")
|
||||
}
|
||||
}
|
||||
|
||||
func fillGradient(img *image.NRGBA) {
|
||||
for y := 0; y < img.Bounds().Dy(); y++ {
|
||||
for x := 0; x < img.Bounds().Dx(); x++ {
|
||||
img.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func encodePNG(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
if err := png.Encode(&output, img); err != nil {
|
||||
t.Fatalf("encode PNG: %v", err)
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
func encodeJPEGForTest(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: 100}); err != nil {
|
||||
t.Fatalf("encode JPEG: %v", err)
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
func injectOrientation(t *testing.T, data []byte, orientation uint16) []byte {
|
||||
t.Helper()
|
||||
if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 {
|
||||
t.Fatal("not a JPEG")
|
||||
}
|
||||
tiff := make([]byte, 26)
|
||||
copy(tiff[:2], "II")
|
||||
binary.LittleEndian.PutUint16(tiff[2:4], 42)
|
||||
binary.LittleEndian.PutUint32(tiff[4:8], 8)
|
||||
binary.LittleEndian.PutUint16(tiff[8:10], 1)
|
||||
binary.LittleEndian.PutUint16(tiff[10:12], 0x0112)
|
||||
binary.LittleEndian.PutUint16(tiff[12:14], 3)
|
||||
binary.LittleEndian.PutUint32(tiff[14:18], 1)
|
||||
binary.LittleEndian.PutUint16(tiff[18:20], orientation)
|
||||
payload := append([]byte("Exif\x00\x00"), tiff...)
|
||||
segment := []byte{0xff, 0xe1, 0, 0}
|
||||
binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2))
|
||||
result := make([]byte, 0, len(data)+len(segment)+len(payload))
|
||||
result = append(result, data[:2]...)
|
||||
result = append(result, segment...)
|
||||
result = append(result, payload...)
|
||||
return append(result, data[2:]...)
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/color"
|
||||
)
|
||||
|
||||
func fit(source image.Image, maxWidth, maxHeight int, upscale bool) *image.NRGBA {
|
||||
bounds := source.Bounds()
|
||||
width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale)
|
||||
return resize(source, width, height)
|
||||
}
|
||||
|
||||
func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) {
|
||||
if !upscale && width <= maxWidth && height <= maxHeight {
|
||||
return width, height
|
||||
}
|
||||
if int64(width)*int64(maxHeight) > int64(height)*int64(maxWidth) {
|
||||
newHeight := max(1, int(int64(height)*int64(maxWidth)/int64(width)))
|
||||
return maxWidth, newHeight
|
||||
}
|
||||
newWidth := max(1, int(int64(width)*int64(maxHeight)/int64(height)))
|
||||
return newWidth, maxHeight
|
||||
}
|
||||
|
||||
func centeredSquare(source image.Image) image.Image {
|
||||
bounds := source.Bounds()
|
||||
size := min(bounds.Dx(), bounds.Dy())
|
||||
left := bounds.Min.X + (bounds.Dx()-size)/2
|
||||
top := bounds.Min.Y + (bounds.Dy()-size)/2
|
||||
return croppedImage{source: source, bounds: image.Rect(0, 0, size, size), offset: image.Pt(left, top)}
|
||||
}
|
||||
|
||||
type croppedImage struct {
|
||||
source image.Image
|
||||
bounds image.Rectangle
|
||||
offset image.Point
|
||||
}
|
||||
|
||||
func (i croppedImage) ColorModel() color.Model { return i.source.ColorModel() }
|
||||
func (i croppedImage) Bounds() image.Rectangle { return i.bounds }
|
||||
func (i croppedImage) At(x, y int) color.Color {
|
||||
return i.source.At(i.offset.X+x, i.offset.Y+y)
|
||||
}
|
||||
|
||||
func resize(source image.Image, width, height int) *image.NRGBA {
|
||||
bounds := source.Bounds()
|
||||
if bounds.Dx() == width && bounds.Dy() == height {
|
||||
return copyNRGBA(source)
|
||||
}
|
||||
output := image.NewNRGBA(image.Rect(0, 0, width, height))
|
||||
for y := 0; y < height; y++ {
|
||||
y0, y1, fy := sampleCoordinate(y, height, bounds.Dy())
|
||||
for x := 0; x < width; x++ {
|
||||
x0, x1, fx := sampleCoordinate(x, width, bounds.Dx())
|
||||
c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA)
|
||||
c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA)
|
||||
c01 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y1)).(color.NRGBA)
|
||||
c11 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y1)).(color.NRGBA)
|
||||
output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy))
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) {
|
||||
coordinate := ((int64(2*destination+1)*int64(sourceSize))<<15)/int64(destinationSize) - 1<<15
|
||||
if coordinate <= 0 {
|
||||
return 0, min(1, sourceSize-1), 0
|
||||
}
|
||||
maximum := int64(sourceSize-1) << 16
|
||||
if coordinate >= maximum {
|
||||
return sourceSize - 1, sourceSize - 1, 0
|
||||
}
|
||||
first := int(coordinate >> 16)
|
||||
return first, first + 1, uint32(coordinate & 0xffff)
|
||||
}
|
||||
|
||||
func interpolate(c00, c10, c01, c11 color.NRGBA, fx, fy uint32) color.NRGBA {
|
||||
return color.NRGBA{
|
||||
R: interpolateChannel(c00.R, c10.R, c01.R, c11.R, fx, fy),
|
||||
G: interpolateChannel(c00.G, c10.G, c01.G, c11.G, fx, fy),
|
||||
B: interpolateChannel(c00.B, c10.B, c01.B, c11.B, fx, fy),
|
||||
A: interpolateChannel(c00.A, c10.A, c01.A, c11.A, fx, fy),
|
||||
}
|
||||
}
|
||||
|
||||
func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 {
|
||||
const one uint64 = 1 << 16
|
||||
top := uint64(c00)*(one-uint64(fx)) + uint64(c10)*uint64(fx)
|
||||
bottom := uint64(c01)*(one-uint64(fx)) + uint64(c11)*uint64(fx)
|
||||
value := top*(one-uint64(fy)) + bottom*uint64(fy)
|
||||
return uint8((value + 1<<31) >> 32)
|
||||
}
|
||||
|
||||
func copyNRGBA(source image.Image) *image.NRGBA {
|
||||
bounds := source.Bounds()
|
||||
output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy()))
|
||||
for y := 0; y < bounds.Dy(); y++ {
|
||||
for x := 0; x < bounds.Dx(); x++ {
|
||||
output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y))
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
@@ -0,0 +1,452 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
AttachmentJob(context.Context, string) (model.AttachmentJob, error)
|
||||
ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error)
|
||||
CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error)
|
||||
RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time) (model.JobMutationResult, error)
|
||||
FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error)
|
||||
RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error)
|
||||
}
|
||||
|
||||
type Broker interface {
|
||||
Read(context.Context) ([]redisstream.Delivery, error)
|
||||
Reclaim(context.Context, string) ([]redisstream.Delivery, string, error)
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
Get(context.Context, string) (io.ReadCloser, error)
|
||||
Put(context.Context, string, string, []byte) error
|
||||
Delete(context.Context, string) error
|
||||
}
|
||||
|
||||
type ImageProcessor interface {
|
||||
Process(context.Context, []byte, string) (transform.Result, error)
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObserveStarted()
|
||||
ObserveFinished(string, time.Duration)
|
||||
ObserveRetry()
|
||||
ObserveReclaimed(int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
WorkerID string
|
||||
Concurrency int
|
||||
LeaseDuration time.Duration
|
||||
ProcessingTimeout time.Duration
|
||||
ReclaimInterval time.Duration
|
||||
MaxAttempts int
|
||||
RetryBase time.Duration
|
||||
RetryMax time.Duration
|
||||
MaxBytes int64
|
||||
MaxPixels int64
|
||||
MemoryBytes int64
|
||||
}
|
||||
|
||||
type FaultPoint string
|
||||
|
||||
const (
|
||||
FaultNone FaultPoint = ""
|
||||
FaultBeforeSideEffect FaultPoint = "before_side_effect"
|
||||
FaultAfterSideEffect FaultPoint = "after_side_effect"
|
||||
FaultAfterCommit FaultPoint = "after_commit"
|
||||
)
|
||||
|
||||
var ErrInjectedFailure = errors.New("injected worker failure")
|
||||
|
||||
type Worker struct {
|
||||
logger *slog.Logger
|
||||
repository Repository
|
||||
broker Broker
|
||||
objects ObjectStore
|
||||
processor ImageProcessor
|
||||
metrics Metrics
|
||||
config Config
|
||||
memory *semaphore.Weighted
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func New(
|
||||
logger *slog.Logger,
|
||||
repository Repository,
|
||||
broker Broker,
|
||||
objects ObjectStore,
|
||||
processor ImageProcessor,
|
||||
metrics Metrics,
|
||||
config Config,
|
||||
) (*Worker, error) {
|
||||
if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil {
|
||||
return nil, errors.New("worker dependencies are required")
|
||||
}
|
||||
if err := validateConfig(config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metrics == nil {
|
||||
metrics = nopMetrics{}
|
||||
}
|
||||
return &Worker{
|
||||
logger: logger, repository: repository, broker: broker, objects: objects,
|
||||
processor: processor, metrics: metrics, config: config,
|
||||
memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateConfig(config Config) error {
|
||||
if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 ||
|
||||
config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration ||
|
||||
config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 ||
|
||||
config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 {
|
||||
return errors.New("invalid worker configuration")
|
||||
}
|
||||
const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128
|
||||
if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 ||
|
||||
config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes {
|
||||
return errors.New("worker memory must fit one maximum image")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) error {
|
||||
lastReclaim := time.Time{}
|
||||
reclaimCursor := "0-0"
|
||||
for ctx.Err() == nil {
|
||||
deliveries, err := w.broker.Read(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
w.processBatch(ctx, deliveries)
|
||||
if w.now().Sub(lastReclaim) < w.config.ReclaimInterval {
|
||||
continue
|
||||
}
|
||||
recovery, err := w.repository.RecoverExpiredAttachmentJobs(
|
||||
ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recover expired attachment jobs: %w", err)
|
||||
}
|
||||
if recovery.Requeued > 0 || recovery.Failed > 0 {
|
||||
w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed)
|
||||
}
|
||||
deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
w.metrics.ObserveReclaimed(len(deliveries))
|
||||
w.processBatch(ctx, deliveries)
|
||||
reclaimCursor = next
|
||||
if reclaimCursor == "" {
|
||||
reclaimCursor = "0-0"
|
||||
}
|
||||
lastReclaim = w.now()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) {
|
||||
limit := make(chan struct{}, w.config.Concurrency)
|
||||
var group sync.WaitGroup
|
||||
for _, delivery := range deliveries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
limit <- struct{}{}
|
||||
group.Add(1)
|
||||
go func(delivery redisstream.Delivery) {
|
||||
defer group.Done()
|
||||
defer func() { <-limit }()
|
||||
if err := w.ProcessDelivery(ctx, delivery, FaultNone); err != nil && ctx.Err() == nil {
|
||||
w.logger.Error("attachment job delivery failed", "message_id", delivery.ID, "error", err)
|
||||
}
|
||||
}(delivery)
|
||||
}
|
||||
group.Wait()
|
||||
}
|
||||
|
||||
func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) error {
|
||||
started := w.now()
|
||||
w.metrics.ObserveStarted()
|
||||
result := "transient_error"
|
||||
defer func() { w.metrics.ObserveFinished(result, w.now().Sub(started)) }()
|
||||
|
||||
if delivery.DecodeError != nil {
|
||||
result = "invalid_job"
|
||||
return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
||||
}
|
||||
envelope := delivery.Envelope
|
||||
lease, err := w.repository.ClaimAttachmentJob(
|
||||
ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt,
|
||||
w.config.WorkerID, w.config.LeaseDuration,
|
||||
)
|
||||
if err != nil {
|
||||
result, err = w.resolveClaim(ctx, delivery, envelope, err)
|
||||
return err
|
||||
}
|
||||
if fault == FaultBeforeSideEffect {
|
||||
result = "injected_failure"
|
||||
return ErrInjectedFailure
|
||||
}
|
||||
|
||||
processingCtx, cancel := context.WithTimeout(ctx, w.config.ProcessingTimeout)
|
||||
variants, err := w.process(processingCtx, lease)
|
||||
cancel()
|
||||
if err != nil {
|
||||
result, err = w.resolveProcessingFailure(ctx, delivery, lease, err)
|
||||
return err
|
||||
}
|
||||
if fault == FaultAfterSideEffect {
|
||||
result = "injected_failure"
|
||||
return ErrInjectedFailure
|
||||
}
|
||||
mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete attachment job: %w", err)
|
||||
}
|
||||
if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") {
|
||||
if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil {
|
||||
w.logger.Warn("staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err)
|
||||
}
|
||||
}
|
||||
if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale {
|
||||
return fmt.Errorf("unexpected completion result %q", mutation)
|
||||
}
|
||||
if fault == FaultAfterCommit {
|
||||
result = "injected_failure"
|
||||
return ErrInjectedFailure
|
||||
}
|
||||
if err := delivery.Ack(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
result = "ready"
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) resolveClaim(
|
||||
ctx context.Context,
|
||||
delivery redisstream.Delivery,
|
||||
envelope redisstream.Envelope,
|
||||
claimErr error,
|
||||
) (string, error) {
|
||||
switch {
|
||||
case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput):
|
||||
return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()})
|
||||
case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline):
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
case errors.Is(claimErr, model.ErrJobBusy):
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
case errors.Is(claimErr, model.ErrJobNotDue):
|
||||
return "not_due", nil
|
||||
case errors.Is(claimErr, model.ErrConflict):
|
||||
job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String())
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("inspect conflicting job: %w", err)
|
||||
}
|
||||
switch job.Status {
|
||||
case model.AttachmentJobReady:
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
case model.AttachmentJobFailed:
|
||||
code := "processing_failed"
|
||||
if job.ErrorCode != nil {
|
||||
code = *job.ErrorCode
|
||||
}
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
default:
|
||||
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
||||
}
|
||||
default:
|
||||
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) {
|
||||
body, err := w.objects.Get(ctx, lease.Attachment.StagingKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = body.Close() }()
|
||||
data, err := io.ReadAll(io.LimitReader(body, w.config.MaxBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > w.config.MaxBytes {
|
||||
return nil, transform.ErrPayloadTooLarge
|
||||
}
|
||||
imageConfig, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, transform.ErrInvalidImage
|
||||
}
|
||||
weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if weight > w.config.MemoryBytes {
|
||||
return nil, errWorkerCapacity
|
||||
}
|
||||
if err := w.memory.Acquire(ctx, weight); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer w.memory.Release(weight)
|
||||
|
||||
processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
variants := []struct {
|
||||
name string
|
||||
value transform.Variant
|
||||
}{
|
||||
{name: "original", value: processed.Original},
|
||||
{name: "thumbnail", value: processed.Thumbnail},
|
||||
{name: "crop", value: processed.Crop},
|
||||
{name: "optimized", value: processed.Optimized},
|
||||
}
|
||||
result := make([]model.Variant, 0, len(variants))
|
||||
for _, item := range variants {
|
||||
objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value)
|
||||
if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, model.Variant{
|
||||
Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)),
|
||||
SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height,
|
||||
ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var errWorkerCapacity = errors.New("image exceeds worker memory budget")
|
||||
|
||||
func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string {
|
||||
extension := ".bin"
|
||||
switch value.ContentType {
|
||||
case "image/png":
|
||||
extension = ".png"
|
||||
case "image/jpeg":
|
||||
extension = ".jpg"
|
||||
case "image/webp":
|
||||
extension = ".webp"
|
||||
}
|
||||
return path.Join(
|
||||
"attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion),
|
||||
name, value.SHA256+extension,
|
||||
)
|
||||
}
|
||||
|
||||
func (w *Worker) resolveProcessingFailure(
|
||||
ctx context.Context,
|
||||
delivery redisstream.Delivery,
|
||||
lease model.AttachmentJobLease,
|
||||
processingErr error,
|
||||
) (string, error) {
|
||||
code, permanent := processingError(processingErr)
|
||||
if permanent || lease.Job.Attempt >= w.config.MaxAttempts {
|
||||
if !permanent {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
mutation, err := w.repository.FailAttachmentJob(ctx, lease, code)
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("fail attachment job: %w", err)
|
||||
}
|
||||
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
}
|
||||
if mutation != model.JobMutationApplied {
|
||||
return "transient_error", fmt.Errorf("unexpected failure result %q", mutation)
|
||||
}
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
}
|
||||
nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt))
|
||||
mutation, err := w.repository.RetryAttachmentJob(ctx, lease, code, nextAttempt)
|
||||
if err != nil {
|
||||
return "transient_error", fmt.Errorf("retry attachment job: %w", err)
|
||||
}
|
||||
if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete {
|
||||
return "duplicate", delivery.Ack(ctx)
|
||||
}
|
||||
if mutation != model.JobMutationApplied {
|
||||
return "transient_error", fmt.Errorf("unexpected retry result %q", mutation)
|
||||
}
|
||||
w.metrics.ObserveRetry()
|
||||
return "retry", delivery.Ack(ctx)
|
||||
}
|
||||
|
||||
func processingError(err error) (string, bool) {
|
||||
switch {
|
||||
case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia):
|
||||
return "invalid_image", true
|
||||
case errors.Is(err, transform.ErrPayloadTooLarge):
|
||||
return "image_too_large", true
|
||||
case errors.Is(err, objectstore.ErrNotFound):
|
||||
return "source_missing", true
|
||||
case errors.Is(err, errWorkerCapacity):
|
||||
return "worker_capacity", true
|
||||
default:
|
||||
return "processing_failed", false
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) retryDelay(jobID string, attempt int) time.Duration {
|
||||
delay := w.config.RetryBase
|
||||
for step := 1; step < attempt && delay < w.config.RetryMax; step++ {
|
||||
if delay > w.config.RetryMax/2 {
|
||||
delay = w.config.RetryMax
|
||||
break
|
||||
}
|
||||
delay *= 2
|
||||
}
|
||||
if delay > w.config.RetryMax {
|
||||
delay = w.config.RetryMax
|
||||
}
|
||||
var hash uint64
|
||||
for index := range jobID {
|
||||
hash = hash*131 + uint64(jobID[index])
|
||||
}
|
||||
jitterRange := min(delay/4, w.config.RetryMax-delay)
|
||||
if jitterRange <= 0 {
|
||||
return delay
|
||||
}
|
||||
jitter := time.Duration(hash % uint64(jitterRange))
|
||||
if delay > time.Duration(math.MaxInt64)-jitter {
|
||||
return delay
|
||||
}
|
||||
return delay + jitter
|
||||
}
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObserveStarted() {}
|
||||
func (nopMetrics) ObserveFinished(string, time.Duration) {}
|
||||
func (nopMetrics) ObserveRetry() {}
|
||||
func (nopMetrics) ObserveReclaimed(int) {}
|
||||
@@ -0,0 +1,56 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
)
|
||||
|
||||
func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) {
|
||||
config := validConfig()
|
||||
require.NoError(t, validateConfig(config))
|
||||
|
||||
config.MemoryBytes = 384 * 1024 * 1024
|
||||
require.ErrorContains(t, validateConfig(config), "memory must fit")
|
||||
}
|
||||
|
||||
func TestRetryDelayIsBounded(t *testing.T) {
|
||||
worker := &Worker{config: Config{RetryBase: time.Second, RetryMax: 4 * time.Second}}
|
||||
for attempt := 1; attempt <= 10; attempt++ {
|
||||
delay := worker.retryDelay("11111111-1111-4111-8111-111111111111", attempt)
|
||||
require.GreaterOrEqual(t, delay, time.Second)
|
||||
require.LessOrEqual(t, delay, 4*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessingErrorClassification(t *testing.T) {
|
||||
tests := []struct {
|
||||
err error
|
||||
code string
|
||||
permanent bool
|
||||
}{
|
||||
{err: transform.ErrInvalidImage, code: "invalid_image", permanent: true},
|
||||
{err: transform.ErrPayloadTooLarge, code: "image_too_large", permanent: true},
|
||||
{err: objectstore.ErrNotFound, code: "source_missing", permanent: true},
|
||||
{err: errors.New("temporary S3 failure"), code: "processing_failed", permanent: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
code, permanent := processingError(test.err)
|
||||
require.Equal(t, test.code, code)
|
||||
require.Equal(t, test.permanent, permanent)
|
||||
}
|
||||
}
|
||||
|
||||
func validConfig() Config {
|
||||
return Config{
|
||||
WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second,
|
||||
ProcessingTimeout: 20 * time.Second, ReclaimInterval: 5 * time.Second,
|
||||
MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second,
|
||||
MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -46,6 +47,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{
|
||||
ChatID: chatID, OwnerID: user.ID, Filename: filename,
|
||||
ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body,
|
||||
Traceparent: traceparent(r.Header.Get("traceparent")), Tracestate: tracestate(r.Header.Get("tracestate")),
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, model.ErrCapacity) {
|
||||
@@ -58,7 +60,7 @@ func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiEr
|
||||
}
|
||||
result = "success"
|
||||
s.metrics.attachmentBytes.Add(float64(uploaded.SizeBytes))
|
||||
writeJSON(w, http.StatusCreated, uploaded)
|
||||
writeJSON(w, http.StatusAccepted, uploaded)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -108,20 +110,21 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
|
||||
return authErr
|
||||
}
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || !originalVariant(query) {
|
||||
variant, valid := attachmentVariant(query)
|
||||
if err != nil || !valid {
|
||||
return invalidRequest()
|
||||
}
|
||||
attachmentID, apiErr := attachmentIDFromRequest(r)
|
||||
if apiErr != nil {
|
||||
return apiErr
|
||||
}
|
||||
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID)
|
||||
content, err := s.attachments.Content(r.Context(), attachmentID, user.ID, variant)
|
||||
if err != nil {
|
||||
return attachmentError(err)
|
||||
}
|
||||
defer func() { _ = content.Body.Close() }()
|
||||
w.Header().Set("Content-Type", content.Attachment.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(content.Attachment.SizeBytes, 10))
|
||||
w.Header().Set("Content-Type", content.Variant.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(content.Variant.SizeBytes, 10))
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename}))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, content.Body); err != nil {
|
||||
@@ -130,11 +133,35 @@ func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiE
|
||||
return nil
|
||||
}
|
||||
|
||||
func originalVariant(query url.Values) bool {
|
||||
func attachmentVariant(query url.Values) (string, bool) {
|
||||
if len(query) == 0 {
|
||||
return true
|
||||
return "original", true
|
||||
}
|
||||
return len(query) == 1 && len(query["variant"]) == 1 && query["variant"][0] == "original"
|
||||
if len(query) != 1 || len(query["variant"]) != 1 {
|
||||
return "", false
|
||||
}
|
||||
switch query["variant"][0] {
|
||||
case "original", "thumbnail", "crop", "optimized":
|
||||
return query["variant"][0], true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
var traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`)
|
||||
|
||||
func traceparent(value string) string {
|
||||
if traceparentPattern.MatchString(value) {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func tracestate(value string) string {
|
||||
if len(value) <= 512 {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func attachmentIDFromRequest(r *http.Request) (string, *apiError) {
|
||||
@@ -158,6 +185,8 @@ func attachmentError(err error) *apiError {
|
||||
return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"}
|
||||
case errors.Is(err, model.ErrCapacity):
|
||||
return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"}
|
||||
case errors.Is(err, model.ErrConflict):
|
||||
return &apiError{http.StatusConflict, errorCodeConflict, "attachment is not ready"}
|
||||
default:
|
||||
return unavailable()
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ type MessageRepository interface {
|
||||
type AttachmentService interface {
|
||||
Upload(context.Context, attachment.Upload) (model.Attachment, error)
|
||||
Metadata(context.Context, string, string) (model.Attachment, error)
|
||||
Content(context.Context, string, string) (attachment.Content, error)
|
||||
Content(context.Context, string, string, string) (attachment.Content, error)
|
||||
}
|
||||
|
||||
type HealthChecker interface {
|
||||
|
||||
@@ -87,8 +87,12 @@ func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model
|
||||
func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) {
|
||||
return s.uploaded, s.err
|
||||
}
|
||||
func (s attachmentServiceStub) Content(context.Context, string, string) (attachment.Content, error) {
|
||||
return attachment.Content{Attachment: s.uploaded, Body: io.NopCloser(strings.NewReader("content"))}, s.err
|
||||
func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) {
|
||||
return attachment.Content{
|
||||
Attachment: s.uploaded,
|
||||
Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes},
|
||||
Body: io.NopCloser(strings.NewReader("content")),
|
||||
}, s.err
|
||||
}
|
||||
|
||||
type sessionStub struct {
|
||||
|
||||
@@ -15,4 +15,10 @@ var (
|
||||
ErrPayloadTooLarge = errors.New("payload too large")
|
||||
ErrCapacity = errors.New("capacity exhausted")
|
||||
ErrUnavailable = errors.New("dependency unavailable")
|
||||
ErrStaleLease = errors.New("stale lease")
|
||||
ErrStaleAttempt = errors.New("stale attempt")
|
||||
ErrStalePipeline = errors.New("stale pipeline")
|
||||
ErrJobBusy = errors.New("job is already leased")
|
||||
ErrJobComplete = errors.New("job is already complete")
|
||||
ErrJobNotDue = errors.New("job is not due")
|
||||
)
|
||||
|
||||
+94
-24
@@ -6,9 +6,21 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
AttachmentPending = "pending"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentDeleting = "deleting"
|
||||
AttachmentPending = "pending"
|
||||
AttachmentQueued = "queued"
|
||||
AttachmentProcessing = "processing"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentFailed = "failed"
|
||||
AttachmentDeleting = "deleting"
|
||||
|
||||
AttachmentJobQueued = "queued"
|
||||
AttachmentJobProcessing = "processing"
|
||||
AttachmentJobReady = "ready"
|
||||
AttachmentJobFailed = "failed"
|
||||
|
||||
JobMutationApplied JobMutationResult = "applied"
|
||||
JobMutationAlreadyComplete JobMutationResult = "already_complete"
|
||||
JobMutationStale JobMutationResult = "stale"
|
||||
)
|
||||
|
||||
const timestampLayout = "2006-01-02T15:04:05.000000Z"
|
||||
@@ -42,21 +54,23 @@ type Message struct {
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
ReadyAt *time.Time `json:"-"`
|
||||
Referenced bool `json:"-"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
StagingKey string `json:"-"`
|
||||
ReadyAt *time.Time `json:"-"`
|
||||
Referenced bool `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type StoredObject struct {
|
||||
@@ -66,12 +80,68 @@ type StoredObject struct {
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
ObjectKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type AttachmentJob struct {
|
||||
ID string
|
||||
AttachmentID string
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
Status string
|
||||
AvailableAt time.Time
|
||||
LeaseToken string
|
||||
LeaseOwner string
|
||||
LeaseUntil *time.Time
|
||||
ErrorCode *string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type AttachmentJobMessage struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
type AttachmentJobLease struct {
|
||||
Job AttachmentJob
|
||||
Attachment Attachment
|
||||
}
|
||||
|
||||
type OutboxEvent struct {
|
||||
ID string
|
||||
Topic string
|
||||
PartitionKey string
|
||||
Payload json.RawMessage
|
||||
JobID string
|
||||
Attempt int
|
||||
AvailableAt time.Time
|
||||
PublishTries int
|
||||
LeaseToken string
|
||||
LeaseUntil *time.Time
|
||||
LastErrorCode *string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type JobMutationResult string
|
||||
|
||||
type RecoveryResult struct {
|
||||
Requeued int
|
||||
Failed int
|
||||
}
|
||||
|
||||
type Position struct {
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
var errLeaseLost = errors.New("outbox lease lost")
|
||||
|
||||
type Store interface {
|
||||
ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error)
|
||||
MarkOutboxPublished(context.Context, string, string) (bool, error)
|
||||
ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error)
|
||||
RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error)
|
||||
}
|
||||
|
||||
type Publisher interface {
|
||||
PublishConfirmed(context.Context, []byte) (string, error)
|
||||
}
|
||||
|
||||
type Metrics interface {
|
||||
ObservePublished()
|
||||
ObservePublishFailure()
|
||||
ObserveRecovered(int)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
BatchSize int
|
||||
LeaseDuration time.Duration
|
||||
PollInterval time.Duration
|
||||
RecoveryInterval time.Duration
|
||||
RetryDelay time.Duration
|
||||
}
|
||||
|
||||
type Dispatcher struct {
|
||||
store Store
|
||||
publisher Publisher
|
||||
metrics Metrics
|
||||
config Config
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func New(store Store, publisher Publisher, metrics Metrics, config Config) (*Dispatcher, error) {
|
||||
if store == nil {
|
||||
return nil, errors.New("outbox store is required")
|
||||
}
|
||||
if publisher == nil {
|
||||
return nil, errors.New("outbox publisher is required")
|
||||
}
|
||||
if err := config.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metrics == nil {
|
||||
metrics = nopMetrics{}
|
||||
}
|
||||
return &Dispatcher{store: store, publisher: publisher, metrics: metrics, config: config, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
switch {
|
||||
case c.BatchSize <= 0:
|
||||
return errors.New("outbox batch size must be positive")
|
||||
case c.LeaseDuration <= 0:
|
||||
return errors.New("outbox lease duration must be positive")
|
||||
case c.PollInterval <= 0:
|
||||
return errors.New("outbox poll interval must be positive")
|
||||
case c.RecoveryInterval <= 0:
|
||||
return errors.New("outbox recovery interval must be positive")
|
||||
case c.RetryDelay <= 0:
|
||||
return errors.New("outbox retry delay must be positive")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) Run(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
pollTicker := time.NewTicker(d.config.PollInterval)
|
||||
recoveryTicker := time.NewTicker(d.config.RecoveryInterval)
|
||||
defer pollTicker.Stop()
|
||||
defer recoveryTicker.Stop()
|
||||
|
||||
if err := d.RecoverExpired(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
for {
|
||||
if _, err := d.DispatchOnce(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-pollTicker.C:
|
||||
case <-recoveryTicker.C:
|
||||
if err := d.RecoverExpired(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) {
|
||||
events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("claim due outbox events: %w", err)
|
||||
}
|
||||
|
||||
published := 0
|
||||
for _, event := range events {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return published, nil
|
||||
}
|
||||
_, err := d.publisher.PublishConfirmed(ctx, event.Payload)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return published, nil
|
||||
}
|
||||
d.metrics.ObservePublishFailure()
|
||||
nextAttemptAt := d.now().Add(d.config.RetryDelay)
|
||||
released, releaseErr := d.store.ReleaseOutboxEvent(
|
||||
ctx, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt,
|
||||
)
|
||||
if releaseErr != nil {
|
||||
return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr)
|
||||
}
|
||||
if !released {
|
||||
return published, fmt.Errorf("release outbox event after publish failure: %w", errLeaseLost)
|
||||
}
|
||||
continue
|
||||
}
|
||||
marked, err := d.store.MarkOutboxPublished(ctx, event.ID, event.LeaseToken)
|
||||
if err != nil {
|
||||
return published, fmt.Errorf("mark outbox event published: %w", err)
|
||||
}
|
||||
if !marked {
|
||||
return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost)
|
||||
}
|
||||
d.metrics.ObservePublished()
|
||||
published++
|
||||
}
|
||||
return published, nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) RecoverExpired(ctx context.Context) error {
|
||||
recovered, err := d.store.RecoverExpiredOutboxEvents(ctx, d.now())
|
||||
if err != nil {
|
||||
return fmt.Errorf("recover expired outbox leases: %w", err)
|
||||
}
|
||||
d.metrics.ObserveRecovered(recovered)
|
||||
return nil
|
||||
}
|
||||
|
||||
type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObservePublished() {}
|
||||
func (nopMetrics) ObservePublishFailure() {}
|
||||
func (nopMetrics) ObserveRecovered(int) {}
|
||||
@@ -0,0 +1,181 @@
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDispatcherPublishesAndMarksEvent(t *testing.T) {
|
||||
store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}}
|
||||
publisher := &publisherStub{messageID: "1-0"}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, published)
|
||||
require.Equal(t, []string{"event-1:lease-1"}, store.marked)
|
||||
}
|
||||
|
||||
func TestDispatcherRetriesSafelyAfterMarkFailure(t *testing.T) {
|
||||
event := model.OutboxEvent{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}
|
||||
store := &storeStub{events: []model.OutboxEvent{event}, markErr: errors.New("database unavailable")}
|
||||
publisher := &publisherStub{messageID: "1-0"}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
|
||||
_, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, publisher.calls)
|
||||
|
||||
store.markErr = nil
|
||||
store.events[0].LeaseToken = "lease-2"
|
||||
publisher.messageID = "2-0"
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, published)
|
||||
require.Equal(t, 2, publisher.calls)
|
||||
require.JSONEq(t, string(event.Payload), string(publisher.payloads[0]))
|
||||
require.JSONEq(t, string(publisher.payloads[0]), string(publisher.payloads[1]))
|
||||
require.Equal(t, []string{"event-1:lease-2"}, store.marked)
|
||||
}
|
||||
|
||||
func TestDispatcherReleasesPublishFailure(t *testing.T) {
|
||||
store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}}
|
||||
publisher := &publisherStub{err: errors.New("redis unavailable")}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, published)
|
||||
require.Equal(t, "event-1:lease-1:broker_publish_failed", store.released)
|
||||
require.Equal(t, dispatcher.now().Add(time.Second), store.nextAttemptAt)
|
||||
}
|
||||
|
||||
func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) {
|
||||
store := &storeStub{
|
||||
events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "stale-lease", Payload: testPayload()}},
|
||||
markLost: true,
|
||||
}
|
||||
dispatcher := newTestDispatcher(t, store, &publisherStub{messageID: "1-0"})
|
||||
|
||||
published, err := dispatcher.DispatchOnce(context.Background())
|
||||
require.ErrorIs(t, err, errLeaseLost)
|
||||
require.Zero(t, published)
|
||||
}
|
||||
|
||||
func TestDispatcherStopsAfterCancellation(t *testing.T) {
|
||||
store := &storeStub{}
|
||||
publisher := &publisherStub{}
|
||||
dispatcher := newTestDispatcher(t, store, publisher)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- dispatcher.Run(ctx) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("dispatcher did not stop after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatcher {
|
||||
t.Helper()
|
||||
dispatcher, err := New(store, publisher, nil, Config{
|
||||
BatchSize: 10,
|
||||
LeaseDuration: time.Minute,
|
||||
PollInterval: time.Millisecond,
|
||||
RecoveryInterval: time.Second,
|
||||
RetryDelay: time.Second,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return dispatcher
|
||||
}
|
||||
|
||||
func testPayload() json.RawMessage {
|
||||
return json.RawMessage(`{
|
||||
"schema_version": 1,
|
||||
"job_id": "11111111-1111-4111-8111-111111111111",
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": "22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-10-22T12:00:00Z"
|
||||
}`)
|
||||
}
|
||||
|
||||
type storeStub struct {
|
||||
mu sync.Mutex
|
||||
events []model.OutboxEvent
|
||||
markErr error
|
||||
markLost bool
|
||||
marked []string
|
||||
released string
|
||||
nextAttemptAt time.Time
|
||||
recoveredCalls int
|
||||
}
|
||||
|
||||
func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]model.OutboxEvent(nil), s.events...), nil
|
||||
}
|
||||
|
||||
func (s *storeStub) MarkOutboxPublished(_ context.Context, id, leaseToken string) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.markErr != nil {
|
||||
return false, s.markErr
|
||||
}
|
||||
if s.markLost {
|
||||
return false, nil
|
||||
}
|
||||
s.marked = append(s.marked, id+":"+leaseToken)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *storeStub) ReleaseOutboxEvent(
|
||||
_ context.Context,
|
||||
id string,
|
||||
leaseToken string,
|
||||
code string,
|
||||
nextAttemptAt time.Time,
|
||||
) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.released = id + ":" + leaseToken + ":" + code
|
||||
s.nextAttemptAt = nextAttemptAt
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.recoveredCalls++
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
type publisherStub struct {
|
||||
mu sync.Mutex
|
||||
messageID string
|
||||
err error
|
||||
calls int
|
||||
payloads [][]byte
|
||||
}
|
||||
|
||||
func (p *publisherStub) PublishConfirmed(_ context.Context, payload []byte) (string, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.calls++
|
||||
p.payloads = append(p.payloads, append([]byte(nil), payload...))
|
||||
return p.messageID, p.err
|
||||
}
|
||||
@@ -0,0 +1,560 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
const attachmentJobTopic = "attachment.jobs"
|
||||
|
||||
const attachmentJobColumns = `
|
||||
id, attachment_id, pipeline_version, attempt, status, available_at,
|
||||
lease_token, lease_owner, lease_until, error_code, created_at, updated_at`
|
||||
|
||||
func (s *Store) QueueAttachmentWithJobAndOutbox(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
stagingKey string,
|
||||
job model.AttachmentJob,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) (model.Attachment, error) {
|
||||
if job.ID == "" || job.PipelineVersion < 1 {
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
if job.Attempt == 0 {
|
||||
job.Attempt = 1
|
||||
}
|
||||
if job.Attempt != 1 {
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if job.CreatedAt.IsZero() {
|
||||
job.CreatedAt = now
|
||||
}
|
||||
if job.AvailableAt.IsZero() {
|
||||
job.AvailableAt = now
|
||||
}
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("begin queue attachment transaction: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
query := `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', pipeline_version = $3, staging_key = $2,
|
||||
ready_at = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND object_key = $2 AND status = 'pending'
|
||||
RETURNING ` + attachmentColumns
|
||||
var attachment model.Attachment
|
||||
err = scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.PipelineVersion), &attachment)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return s.alreadyQueuedAttachment(ctx, tx, attachmentID, stagingKey, job)
|
||||
}
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("queue attachment: %w", err)
|
||||
}
|
||||
|
||||
job.AttachmentID = attachmentID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = job.CreatedAt
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("commit queued attachment: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) QueueAttachmentPipelineVersion(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
job model.AttachmentJob,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) (model.Attachment, error) {
|
||||
if job.ID == "" || job.PipelineVersion < 2 {
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
job.Attempt = 1
|
||||
now := time.Now().UTC()
|
||||
job.CreatedAt = now
|
||||
job.AvailableAt = now
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("begin queue attachment pipeline version: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
const selectAttachment = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 FOR UPDATE`
|
||||
var attachment model.Attachment
|
||||
if err := scanAttachment(tx.QueryRow(ctx, selectAttachment, attachmentID), &attachment); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrNotFound
|
||||
}
|
||||
return model.Attachment{}, fmt.Errorf("lock attachment for pipeline update: %w", err)
|
||||
}
|
||||
if attachment.Status != model.AttachmentReady || job.PipelineVersion != attachment.PipelineVersion+1 {
|
||||
return model.Attachment{}, ErrConflict
|
||||
}
|
||||
job.AttachmentID = attachment.ID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = now
|
||||
const queueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', pipeline_version = $2, staging_key = object_key,
|
||||
ready_at = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1
|
||||
RETURNING ` + attachmentColumns
|
||||
if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err)
|
||||
}
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("commit queued attachment pipeline version: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) alreadyQueuedAttachment(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
attachmentID string,
|
||||
stagingKey string,
|
||||
job model.AttachmentJob,
|
||||
) (model.Attachment, error) {
|
||||
query := `
|
||||
SELECT ` + attachmentColumnsWithAlias("a") + `
|
||||
FROM attachments a
|
||||
JOIN attachment_jobs j
|
||||
ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
|
||||
WHERE a.id = $1 AND a.staging_key = $2 AND a.status IN ('queued', 'processing', 'ready')
|
||||
AND j.id = $3 AND j.pipeline_version = $4`
|
||||
var attachment model.Attachment
|
||||
if err := scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.ID, job.PipelineVersion), &attachment); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, ErrConflict
|
||||
}
|
||||
return model.Attachment{}, fmt.Errorf("find queued attachment: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("commit repeated queue attachment: %w", err)
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error {
|
||||
const query = `
|
||||
INSERT INTO attachment_jobs (
|
||||
id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt,
|
||||
); err != nil {
|
||||
return fmt.Errorf("insert attachment job: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertAttachmentJobOutbox(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
job model.AttachmentJob,
|
||||
traceparent string,
|
||||
tracestate string,
|
||||
) error {
|
||||
payload, err := json.Marshal(model.AttachmentJobMessage{
|
||||
SchemaVersion: 1, JobID: job.ID, Type: "attachment.process.v1",
|
||||
AttachmentID: job.AttachmentID, PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt, CreatedAt: job.CreatedAt,
|
||||
Traceparent: traceparent, Tracestate: tracestate,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal attachment job event: %w", err)
|
||||
}
|
||||
const query = `
|
||||
INSERT INTO outbox_events (
|
||||
id, topic, partition_key, payload, job_id, attempt, available_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, query, uuid.NewString(), attachmentJobTopic, job.AttachmentID, payload,
|
||||
job.ID, job.Attempt, job.AvailableAt,
|
||||
); err != nil {
|
||||
return fmt.Errorf("insert attachment job outbox event: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentJob(ctx context.Context, jobID string) (model.AttachmentJob, error) {
|
||||
const query = `SELECT ` + attachmentJobColumns + ` FROM attachment_jobs WHERE id = $1`
|
||||
var job model.AttachmentJob
|
||||
if err := scanAttachmentJob(s.pool.QueryRow(ctx, query, jobID), &job); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.AttachmentJob{}, ErrNotFound
|
||||
}
|
||||
return model.AttachmentJob{}, fmt.Errorf("get attachment job: %w", err)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (s *Store) ClaimAttachmentJob(
|
||||
ctx context.Context,
|
||||
jobID string,
|
||||
attachmentID string,
|
||||
pipelineVersion int,
|
||||
attempt int,
|
||||
workerID string,
|
||||
leaseDuration time.Duration,
|
||||
) (model.AttachmentJobLease, error) {
|
||||
if leaseDuration <= 0 || workerID == "" {
|
||||
return model.AttachmentJobLease{}, model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("begin claim attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
||||
if err != nil {
|
||||
return model.AttachmentJobLease{}, err
|
||||
}
|
||||
if attachment.ID != attachmentID || job.AttachmentID != attachmentID {
|
||||
return model.AttachmentJobLease{}, model.ErrInvalidInput
|
||||
}
|
||||
if attachment.PipelineVersion != pipelineVersion || job.PipelineVersion != pipelineVersion {
|
||||
return model.AttachmentJobLease{}, model.ErrStalePipeline
|
||||
}
|
||||
if job.Status == model.AttachmentJobReady || attachment.Status == model.AttachmentReady {
|
||||
return model.AttachmentJobLease{}, model.ErrJobComplete
|
||||
}
|
||||
if job.Attempt != attempt {
|
||||
return model.AttachmentJobLease{}, model.ErrStaleAttempt
|
||||
}
|
||||
if job.Status == model.AttachmentJobProcessing {
|
||||
return model.AttachmentJobLease{}, model.ErrJobBusy
|
||||
}
|
||||
if job.Status != model.AttachmentJobQueued {
|
||||
return model.AttachmentJobLease{}, ErrConflict
|
||||
}
|
||||
if job.AvailableAt.After(time.Now()) {
|
||||
return model.AttachmentJobLease{}, model.ErrJobNotDue
|
||||
}
|
||||
|
||||
job.LeaseToken = uuid.NewString()
|
||||
job.LeaseOwner = workerID
|
||||
leaseUntil := time.Now().UTC().Add(leaseDuration)
|
||||
job.LeaseUntil = &leaseUntil
|
||||
job.Status = model.AttachmentJobProcessing
|
||||
const updateJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing', lease_token = $2, lease_owner = $3,
|
||||
lease_until = $4, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, updateJob, job.ID, job.LeaseToken, job.LeaseOwner, leaseUntil); err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("claim attachment job: %w", err)
|
||||
}
|
||||
const updateAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'processing', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2 AND status = 'queued'`
|
||||
result, err := tx.Exec(ctx, updateAttachment, attachment.ID, pipelineVersion)
|
||||
if err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("mark attachment processing: %w", err)
|
||||
}
|
||||
if result.RowsAffected() != 1 {
|
||||
return model.AttachmentJobLease{}, model.ErrStalePipeline
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.AttachmentJobLease{}, fmt.Errorf("commit attachment job claim: %w", err)
|
||||
}
|
||||
attachment.Status = model.AttachmentProcessing
|
||||
attachment.ErrorCode = nil
|
||||
return model.AttachmentJobLease{Job: job, Attachment: attachment}, nil
|
||||
}
|
||||
|
||||
func lockAttachmentJob(ctx context.Context, tx pgx.Tx, jobID string) (model.AttachmentJob, model.Attachment, error) {
|
||||
query := `
|
||||
SELECT ` + prefixedAttachmentJobColumns("j") + `, ` + attachmentColumnsWithAlias("a") + `
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = $1
|
||||
FOR UPDATE OF j, a`
|
||||
var job model.AttachmentJob
|
||||
var attachment model.Attachment
|
||||
var leaseToken, leaseOwner *string
|
||||
var stagingKey *string
|
||||
err := tx.QueryRow(ctx, query, jobID).Scan(
|
||||
&job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt,
|
||||
&leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt,
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.AttachmentJob{}, model.Attachment{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, model.Attachment{}, fmt.Errorf("lock attachment job: %w", err)
|
||||
}
|
||||
if leaseToken != nil {
|
||||
job.LeaseToken = *leaseToken
|
||||
}
|
||||
if leaseOwner != nil {
|
||||
job.LeaseOwner = *leaseOwner
|
||||
}
|
||||
if stagingKey != nil {
|
||||
attachment.StagingKey = *stagingKey
|
||||
}
|
||||
return job, attachment, nil
|
||||
}
|
||||
|
||||
func prefixedAttachmentJobColumns(alias string) string {
|
||||
return fmt.Sprintf(
|
||||
"%[1]s.id, %[1]s.attachment_id, %[1]s.pipeline_version, %[1]s.attempt, %[1]s.status, "+
|
||||
"%[1]s.available_at, %[1]s.lease_token, %[1]s.lease_owner, %[1]s.lease_until, "+
|
||||
"%[1]s.error_code, %[1]s.created_at, %[1]s.updated_at",
|
||||
alias,
|
||||
)
|
||||
}
|
||||
|
||||
func scanAttachmentJob(row rowScanner, job *model.AttachmentJob) error {
|
||||
var leaseToken, leaseOwner *string
|
||||
err := row.Scan(
|
||||
&job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt,
|
||||
&leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt,
|
||||
)
|
||||
if leaseToken != nil {
|
||||
job.LeaseToken = *leaseToken
|
||||
}
|
||||
if leaseOwner != nil {
|
||||
job.LeaseOwner = *leaseOwner
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) CompleteAttachmentJob(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
variants []model.Variant,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validCompletedVariants(variants) {
|
||||
return "", model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin complete attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
|
||||
if err != nil || result != model.JobMutationApplied {
|
||||
return result, err
|
||||
}
|
||||
const deleteVariants = `
|
||||
DELETE FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, deleteVariants, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return "", fmt.Errorf("replace attachment variants: %w", err)
|
||||
}
|
||||
|
||||
var originalKey string
|
||||
const insertVariant = `
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id, pipeline_version, name, content_type, size_bytes,
|
||||
sha256, width, height, object_key
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`
|
||||
for _, variant := range variants {
|
||||
if variant.Name == "original" {
|
||||
originalKey = variant.ObjectKey
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
ctx, insertVariant, attachment.ID, job.PipelineVersion, variant.Name,
|
||||
variant.ContentType, variant.SizeBytes, variant.SHA256, variant.Width,
|
||||
variant.Height, variant.ObjectKey,
|
||||
); err != nil {
|
||||
return "", fmt.Errorf("insert attachment variant %s: %w", variant.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
const completeJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2`
|
||||
if _, err := tx.Exec(ctx, completeJob, job.ID, lease.Job.LeaseToken); err != nil {
|
||||
return "", fmt.Errorf("complete attachment job: %w", err)
|
||||
}
|
||||
const completeAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'ready', object_key = $3, ready_at = clock_timestamp(),
|
||||
staging_key = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil {
|
||||
return "", fmt.Errorf("complete attachment: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit completed attachment job: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func validCompletedVariants(variants []model.Variant) bool {
|
||||
seen := make(map[string]struct{}, len(variants))
|
||||
for _, variant := range variants {
|
||||
if variant.Name == "" || variant.ObjectKey == "" {
|
||||
return false
|
||||
}
|
||||
if _, duplicate := seen[variant.Name]; duplicate {
|
||||
return false
|
||||
}
|
||||
seen[variant.Name] = struct{}{}
|
||||
}
|
||||
_, hasOriginal := seen["original"]
|
||||
_, hasThumbnail := seen["thumbnail"]
|
||||
return hasOriginal && hasThumbnail
|
||||
}
|
||||
|
||||
func (s *Store) RetryAttachmentJob(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
errorCode string,
|
||||
nextAttemptAt time.Time,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
|
||||
return "", model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin retry attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
|
||||
if err != nil || result != model.JobMutationApplied {
|
||||
return result, err
|
||||
}
|
||||
job.Attempt++
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.AvailableAt = nextAttemptAt.UTC()
|
||||
job.ErrorCode = &errorCode
|
||||
const retryJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $3, available_at = $4,
|
||||
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = $5, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2`
|
||||
if _, err := tx.Exec(
|
||||
ctx, retryJob, job.ID, lease.Job.LeaseToken, job.Attempt, job.AvailableAt, errorCode,
|
||||
); err != nil {
|
||||
return "", fmt.Errorf("retry attachment job: %w", err)
|
||||
}
|
||||
const queueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return "", fmt.Errorf("requeue attachment: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit attachment job retry: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func (s *Store) FailAttachmentJob(
|
||||
ctx context.Context,
|
||||
lease model.AttachmentJobLease,
|
||||
errorCode string,
|
||||
) (model.JobMutationResult, error) {
|
||||
if !validErrorCode(errorCode) {
|
||||
return "", model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin fail attachment job: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
job, attachment, result, err := lockJobMutation(ctx, tx, lease)
|
||||
if err != nil || result != model.JobMutationApplied {
|
||||
return result, err
|
||||
}
|
||||
const failJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = $3, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND lease_token = $2`
|
||||
if _, err := tx.Exec(ctx, failJob, job.ID, lease.Job.LeaseToken, errorCode); err != nil {
|
||||
return "", fmt.Errorf("fail attachment job: %w", err)
|
||||
}
|
||||
const failAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil {
|
||||
return "", fmt.Errorf("fail attachment: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit failed attachment job: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func lockJobMutation(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
lease model.AttachmentJobLease,
|
||||
) (model.AttachmentJob, model.Attachment, model.JobMutationResult, error) {
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, lease.Job.ID)
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, model.Attachment{}, "", err
|
||||
}
|
||||
if attachment.PipelineVersion != lease.Job.PipelineVersion || job.PipelineVersion != lease.Job.PipelineVersion {
|
||||
return job, attachment, model.JobMutationStale, nil
|
||||
}
|
||||
if job.Status == model.AttachmentJobReady && attachment.Status == model.AttachmentReady {
|
||||
return job, attachment, model.JobMutationAlreadyComplete, nil
|
||||
}
|
||||
if job.PipelineVersion != lease.Job.PipelineVersion ||
|
||||
job.Attempt != lease.Job.Attempt ||
|
||||
job.Status != model.AttachmentJobProcessing ||
|
||||
job.LeaseToken == "" || job.LeaseToken != lease.Job.LeaseToken ||
|
||||
job.LeaseUntil == nil || !job.LeaseUntil.After(time.Now()) {
|
||||
return job, attachment, model.JobMutationStale, nil
|
||||
}
|
||||
return job, attachment, model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func validErrorCode(errorCode string) bool {
|
||||
return len(errorCode) >= 1 && len(errorCode) <= 64
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) RecoverExpiredAttachmentJobs(
|
||||
ctx context.Context,
|
||||
limit int,
|
||||
maxAttempts int,
|
||||
retryAt time.Time,
|
||||
) (model.RecoveryResult, error) {
|
||||
if limit < 1 || limit > 1000 || maxAttempts < 1 {
|
||||
return model.RecoveryResult{}, model.ErrInvalidInput
|
||||
}
|
||||
if retryAt.IsZero() {
|
||||
retryAt = time.Now().UTC()
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
const selectExpired = `
|
||||
SELECT id
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'processing' AND lease_until <= clock_timestamp()
|
||||
ORDER BY lease_until, id
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED`
|
||||
rows, err := tx.Query(ctx, selectExpired, limit)
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err)
|
||||
}
|
||||
jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err)
|
||||
}
|
||||
|
||||
var recovered model.RecoveryResult
|
||||
for _, jobID := range jobIDs {
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
||||
if err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) {
|
||||
continue
|
||||
}
|
||||
if job.PipelineVersion != attachment.PipelineVersion {
|
||||
if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Failed++
|
||||
continue
|
||||
}
|
||||
if job.Attempt >= maxAttempts {
|
||||
if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Failed++
|
||||
continue
|
||||
}
|
||||
|
||||
job.Attempt++
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.AvailableAt = retryAt.UTC()
|
||||
const requeueJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $2, available_at = $3,
|
||||
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = 'lease_expired', updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err)
|
||||
}
|
||||
const requeueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Requeued++
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err)
|
||||
}
|
||||
return recovered, nil
|
||||
}
|
||||
|
||||
func markExpiredJobFailed(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
job model.AttachmentJob,
|
||||
attachment model.Attachment,
|
||||
errorCode string,
|
||||
failAttachment bool,
|
||||
) error {
|
||||
const failJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = $2, updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
|
||||
return fmt.Errorf("fail expired attachment job: %w", err)
|
||||
}
|
||||
if !failAttachment {
|
||||
return nil
|
||||
}
|
||||
const failAttachmentQuery = `
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil {
|
||||
return fmt.Errorf("fail attachment after expired lease: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ReplayFailedAttachment(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
actor string,
|
||||
reason string,
|
||||
) (model.AttachmentJob, error) {
|
||||
if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 {
|
||||
return model.AttachmentJob{}, model.ErrInvalidInput
|
||||
}
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
const selectJob = `
|
||||
SELECT j.id
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed'
|
||||
FOR UPDATE OF j, a`
|
||||
var jobID string
|
||||
if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.AttachmentJob{}, ErrConflict
|
||||
}
|
||||
return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err)
|
||||
}
|
||||
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
||||
if err != nil {
|
||||
return model.AttachmentJob{}, err
|
||||
}
|
||||
previousAttempt := job.Attempt
|
||||
job.Attempt++
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.AvailableAt = time.Now().UTC()
|
||||
job.ErrorCode = nil
|
||||
job.LeaseToken = ""
|
||||
job.LeaseOwner = ""
|
||||
job.LeaseUntil = nil
|
||||
const replayJob = `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = $2, available_at = $3,
|
||||
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1`
|
||||
if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err)
|
||||
}
|
||||
const replayAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1 AND pipeline_version = $2`
|
||||
if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err)
|
||||
}
|
||||
const insertAudit = `
|
||||
INSERT INTO attachment_replay_audit (
|
||||
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID,
|
||||
previousAttempt, job.Attempt, actor, reason,
|
||||
); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
return model.AttachmentJob{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
const attachmentColumns = `
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, status, created_at, updated_at, ready_at`
|
||||
object_key, status, created_at, updated_at, ready_at, pipeline_version, error_code, staging_key`
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(...any) error
|
||||
@@ -74,7 +74,7 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri
|
||||
SELECT ` + attachmentColumnsWithAlias("a") + `
|
||||
FROM attachments a
|
||||
JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2
|
||||
WHERE a.id = $1 AND a.status = 'ready'`
|
||||
WHERE a.id = $1 AND a.status IN ('queued', 'processing', 'ready', 'failed')`
|
||||
|
||||
var attachment model.Attachment
|
||||
err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment)
|
||||
@@ -84,9 +84,68 @@ func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID stri
|
||||
if err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err)
|
||||
}
|
||||
if attachment.Status == model.AttachmentReady {
|
||||
attachment.Variants, err = s.attachmentVariants(ctx, attachment.ID, attachment.PipelineVersion)
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
}
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentVariantForUser(
|
||||
ctx context.Context,
|
||||
attachmentID string,
|
||||
userID string,
|
||||
variantName string,
|
||||
) (model.Attachment, model.Variant, error) {
|
||||
attachment, err := s.AttachmentForUser(ctx, attachmentID, userID)
|
||||
if err != nil {
|
||||
return model.Attachment{}, model.Variant{}, err
|
||||
}
|
||||
if attachment.Status != model.AttachmentReady {
|
||||
return model.Attachment{}, model.Variant{}, ErrConflict
|
||||
}
|
||||
for _, variant := range attachment.Variants {
|
||||
if variant.Name == variantName {
|
||||
return attachment, variant, nil
|
||||
}
|
||||
}
|
||||
if variantName == "original" && attachment.ObjectKey != "" {
|
||||
return attachment, model.Variant{
|
||||
Name: "original", ContentType: attachment.ContentType, SizeBytes: attachment.SizeBytes,
|
||||
SHA256: attachment.SHA256, ObjectKey: attachment.ObjectKey,
|
||||
PipelineVersion: attachment.PipelineVersion,
|
||||
}, nil
|
||||
}
|
||||
return model.Attachment{}, model.Variant{}, ErrNotFound
|
||||
}
|
||||
|
||||
func (s *Store) attachmentVariants(ctx context.Context, attachmentID string, pipelineVersion int) ([]model.Variant, error) {
|
||||
const query = `
|
||||
SELECT name, content_type, size_bytes, sha256, width, height, object_key, pipeline_version
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2
|
||||
ORDER BY CASE name
|
||||
WHEN 'original' THEN 0
|
||||
WHEN 'thumbnail' THEN 1
|
||||
WHEN 'crop' THEN 2
|
||||
WHEN 'optimized' THEN 3
|
||||
ELSE 4
|
||||
END`
|
||||
rows, err := s.pool.Query(ctx, query, attachmentID, pipelineVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list attachment variants: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
variants, err := pgx.CollectRows(rows, pgx.RowToStructByPos[model.Variant])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan attachment variants: %w", err)
|
||||
}
|
||||
return variants, nil
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) {
|
||||
const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2`
|
||||
|
||||
@@ -102,7 +161,12 @@ func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
|
||||
const query = `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`
|
||||
const query = `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1
|
||||
UNION ALL
|
||||
SELECT 1 FROM attachment_variants WHERE object_key = $1
|
||||
)`
|
||||
var exists bool
|
||||
if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil {
|
||||
return false, fmt.Errorf("check attachment by object key: %w", err)
|
||||
@@ -144,12 +208,18 @@ func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limi
|
||||
}
|
||||
|
||||
func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error {
|
||||
return row.Scan(
|
||||
var stagingKey *string
|
||||
err := row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
|
||||
&attachment.Referenced,
|
||||
)
|
||||
if stagingKey != nil {
|
||||
attachment.StagingKey = *stagingKey
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) {
|
||||
@@ -232,23 +302,36 @@ func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID str
|
||||
}
|
||||
|
||||
func scanAttachment(row rowScanner, attachment *model.Attachment) error {
|
||||
return row.Scan(
|
||||
var stagingKey *string
|
||||
err := row.Scan(
|
||||
&attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename,
|
||||
&attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey,
|
||||
&attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt,
|
||||
&attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey,
|
||||
)
|
||||
if stagingKey != nil {
|
||||
attachment.StagingKey = *stagingKey
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func attachmentColumnsWithAlias(alias string) string {
|
||||
return fmt.Sprintf(
|
||||
"%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+
|
||||
"%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+
|
||||
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at",
|
||||
"%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at, %[1]s.pipeline_version, "+
|
||||
"%[1]s.error_code, %[1]s.staging_key",
|
||||
alias,
|
||||
)
|
||||
}
|
||||
|
||||
func publicAttachment(attachment model.Attachment) model.Attachment {
|
||||
attachment.Variants = []model.Variant{}
|
||||
variants := make([]model.Variant, 0, len(attachment.Variants))
|
||||
for _, variant := range attachment.Variants {
|
||||
if variant.Name != "original" {
|
||||
variants = append(variants, variant)
|
||||
}
|
||||
}
|
||||
attachment.Variants = variants
|
||||
return attachment
|
||||
}
|
||||
|
||||
@@ -8,4 +8,10 @@ var (
|
||||
ErrInvalidMembers = model.ErrInvalidMembers
|
||||
ErrInvalidCursor = model.ErrInvalidCursor
|
||||
ErrIdempotencyConflict = model.ErrIdempotencyConflict
|
||||
ErrStaleLease = model.ErrStaleLease
|
||||
ErrStaleAttempt = model.ErrStaleAttempt
|
||||
ErrStalePipeline = model.ErrStalePipeline
|
||||
ErrJobBusy = model.ErrJobBusy
|
||||
ErrJobComplete = model.ErrJobComplete
|
||||
ErrJobNotDue = model.ErrJobNotDue
|
||||
)
|
||||
|
||||
@@ -51,13 +51,16 @@ func (s *Store) CreateMessage(
|
||||
if !visible {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
validAttachments, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
|
||||
validAttachments, failedAttachment, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs)
|
||||
if err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
if !validAttachments {
|
||||
return model.Message{}, false, ErrNotFound
|
||||
}
|
||||
if failedAttachment {
|
||||
return model.Message{}, false, ErrConflict
|
||||
}
|
||||
if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil {
|
||||
return model.Message{}, false, err
|
||||
}
|
||||
@@ -92,28 +95,35 @@ func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte,
|
||||
return sha256.Sum256(payload), nil
|
||||
}
|
||||
|
||||
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, error) {
|
||||
func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, bool, error) {
|
||||
if len(attachmentIDs) == 0 {
|
||||
return true, nil
|
||||
return true, false, nil
|
||||
}
|
||||
const query = `
|
||||
SELECT id
|
||||
SELECT id, status
|
||||
FROM attachments
|
||||
WHERE chat_id = $1 AND id::text = ANY($2) AND status = 'ready'
|
||||
WHERE chat_id = $1 AND id::text = ANY($2)
|
||||
AND status IN ('queued', 'processing', 'ready', 'failed')
|
||||
FOR UPDATE`
|
||||
rows, err := tx.Query(ctx, query, chatID, attachmentIDs)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||
return false, false, fmt.Errorf("validate message attachments: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
count := 0
|
||||
var count int
|
||||
var failed bool
|
||||
for rows.Next() {
|
||||
var id, status string
|
||||
if err := rows.Scan(&id, &status); err != nil {
|
||||
return false, false, fmt.Errorf("scan message attachment: %w", err)
|
||||
}
|
||||
count++
|
||||
failed = failed || status == model.AttachmentFailed
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return false, fmt.Errorf("validate message attachments: %w", err)
|
||||
return false, false, fmt.Errorf("validate message attachments: %w", err)
|
||||
}
|
||||
return count == len(attachmentIDs), nil
|
||||
return count == len(attachmentIDs), failed, nil
|
||||
}
|
||||
|
||||
func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) {
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
func (s *Store) ClaimOutboxEvents(
|
||||
ctx context.Context,
|
||||
limit int,
|
||||
leaseDuration time.Duration,
|
||||
) ([]model.OutboxEvent, error) {
|
||||
if limit < 1 || limit > 1000 || leaseDuration <= 0 {
|
||||
return nil, model.ErrInvalidInput
|
||||
}
|
||||
claimToken := uuid.NewString()
|
||||
query := `
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM outbox_events
|
||||
WHERE published_at IS NULL
|
||||
AND available_at <= clock_timestamp()
|
||||
AND (claim_until IS NULL OR claim_until <= clock_timestamp())
|
||||
ORDER BY available_at, id
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE outbox_events event
|
||||
SET claim_token = $2,
|
||||
claim_until = clock_timestamp() + $3 * INTERVAL '1 millisecond',
|
||||
publish_tries = event.publish_tries + 1
|
||||
FROM candidates
|
||||
WHERE event.id = candidates.id
|
||||
RETURNING event.id, event.topic, event.partition_key, event.payload, event.job_id,
|
||||
event.attempt, event.available_at, event.publish_tries, event.claim_token,
|
||||
event.claim_until, event.last_error_code, event.created_at`
|
||||
rows, err := s.pool.Query(ctx, query, limit, claimToken, leaseDuration.Milliseconds())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("claim outbox events: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
events, err := pgx.CollectRows(rows, pgx.RowToFunc[model.OutboxEvent](func(row pgx.CollectableRow) (model.OutboxEvent, error) {
|
||||
var event model.OutboxEvent
|
||||
err := row.Scan(
|
||||
&event.ID, &event.Topic, &event.PartitionKey, &event.Payload, &event.JobID,
|
||||
&event.Attempt, &event.AvailableAt, &event.PublishTries, &event.LeaseToken,
|
||||
&event.LeaseUntil, &event.LastErrorCode, &event.CreatedAt,
|
||||
)
|
||||
return event, err
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan claimed outbox events: %w", err)
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func (s *Store) MarkOutboxPublished(ctx context.Context, eventID, leaseToken string) (bool, error) {
|
||||
const query = `
|
||||
UPDATE outbox_events
|
||||
SET published_at = clock_timestamp(), claim_token = NULL, claim_until = NULL,
|
||||
last_error_code = NULL
|
||||
WHERE id = $1 AND claim_token = $2 AND published_at IS NULL
|
||||
AND claim_until > clock_timestamp()`
|
||||
result, err := s.pool.Exec(ctx, query, eventID, leaseToken)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("mark outbox event published: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
func (s *Store) ReleaseOutboxEvent(
|
||||
ctx context.Context,
|
||||
eventID string,
|
||||
leaseToken string,
|
||||
errorCode string,
|
||||
nextAttemptAt time.Time,
|
||||
) (bool, error) {
|
||||
if !validErrorCode(errorCode) || nextAttemptAt.IsZero() {
|
||||
return false, model.ErrInvalidInput
|
||||
}
|
||||
const query = `
|
||||
UPDATE outbox_events
|
||||
SET available_at = $3, claim_token = NULL, claim_until = NULL, last_error_code = $4
|
||||
WHERE id = $1 AND claim_token = $2 AND published_at IS NULL
|
||||
AND claim_until > clock_timestamp()`
|
||||
result, err := s.pool.Exec(ctx, query, eventID, leaseToken, nextAttemptAt.UTC(), errorCode)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("release outbox event: %w", err)
|
||||
}
|
||||
return result.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (int, error) {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
const query = `
|
||||
UPDATE outbox_events
|
||||
SET claim_token = NULL, claim_until = NULL
|
||||
WHERE published_at IS NULL AND claim_until <= $1`
|
||||
result, err := s.pool.Exec(ctx, query, now.UTC())
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("recover expired outbox events: %w", err)
|
||||
}
|
||||
return int(result.RowsAffected()), nil
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -159,6 +160,165 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
|
||||
s.False(deleted)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
s.Equal(model.AttachmentQueued, attachment.Status)
|
||||
|
||||
events, err := s.store.ClaimOutboxEvents(context.Background(), 10, time.Minute)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotEmpty(events)
|
||||
var event model.OutboxEvent
|
||||
for _, candidate := range events {
|
||||
if candidate.JobID == job.ID {
|
||||
event = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
s.Require().NotEmpty(event.ID)
|
||||
var message model.AttachmentJobMessage
|
||||
s.Require().NoError(json.Unmarshal(event.Payload, &message))
|
||||
s.Equal(job.ID, message.JobID)
|
||||
s.Equal(attachment.ID, message.AttachmentID)
|
||||
s.Equal(1, message.Attempt)
|
||||
published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken)
|
||||
s.Require().NoError(err)
|
||||
s.True(published)
|
||||
|
||||
_, replay, err := s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "queued attachment",
|
||||
[]string{attachment.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.False(replay)
|
||||
|
||||
_, err = s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, uuid.NewString(), job.PipelineVersion, job.Attempt, "worker-invalid", time.Minute,
|
||||
)
|
||||
s.ErrorIs(err, model.ErrInvalidInput)
|
||||
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentProcessing, lease.Attachment.Status)
|
||||
s.NotEmpty(lease.Attachment.StagingKey)
|
||||
|
||||
variants := []model.Variant{
|
||||
{
|
||||
Name: "original", ContentType: "image/png", SizeBytes: 128,
|
||||
SHA256: strings.Repeat("a", 64), Width: 320, Height: 200,
|
||||
ObjectKey: "attachments/" + attachment.ID + "/v1/original.png",
|
||||
},
|
||||
{
|
||||
Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64,
|
||||
SHA256: strings.Repeat("b", 64), Width: 256, Height: 160,
|
||||
ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg",
|
||||
},
|
||||
}
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationAlreadyComplete, result)
|
||||
|
||||
metadata, variant, err := s.store.AttachmentVariantForUser(
|
||||
context.Background(), attachment.ID, owner.ID, "thumbnail",
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentReady, metadata.Status)
|
||||
s.Len(metadata.Variants, 1)
|
||||
s.Equal(variants[1].ObjectKey, variant.ObjectKey)
|
||||
|
||||
v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2}
|
||||
queuedV2, err := s.store.QueueAttachmentPipelineVersion(
|
||||
context.Background(), attachment.ID, v2, "", "",
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentQueued, queuedV2.Status)
|
||||
s.Equal(2, queuedV2.PipelineVersion)
|
||||
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image")
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
|
||||
_, _, err = s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "failed attachment",
|
||||
[]string{attachment.ID}, uuid.NewString(), 24*time.Hour,
|
||||
)
|
||||
s.ErrorIs(err, postgres.ErrConflict)
|
||||
metadata, err := s.store.AttachmentForUser(context.Background(), attachment.ID, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentFailed, metadata.Status)
|
||||
s.Require().NotNil(metadata.ErrorCode)
|
||||
s.Equal("invalid_image", *metadata.ErrorCode)
|
||||
|
||||
replayed, err := s.store.ReplayFailedAttachment(
|
||||
context.Background(), attachment.ID, "operator@example", "input repaired",
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, replayed.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, replayed.Status)
|
||||
|
||||
var auditCount int
|
||||
err = s.pool.QueryRow(
|
||||
context.Background(), `SELECT count(*) FROM attachment_replay_audit WHERE job_id = $1`, job.ID,
|
||||
).Scan(&auditCount)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, auditCount)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.pool.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
|
||||
job.ID,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 5, time.Now())
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, recovered.Requeued)
|
||||
s.Zero(recovered.Failed)
|
||||
current, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, current.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, current.Status)
|
||||
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, []model.Variant{
|
||||
{
|
||||
Name: "original", ContentType: "image/png", SizeBytes: 128,
|
||||
SHA256: strings.Repeat("a", 64), Width: 320, Height: 200,
|
||||
ObjectKey: "attachments/" + attachment.ID + "/v1/original.png",
|
||||
},
|
||||
{
|
||||
Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 64,
|
||||
SHA256: strings.Repeat("b", 64), Width: 256, Height: 160,
|
||||
ObjectKey: "attachments/" + attachment.ID + "/v1/thumbnail.jpg",
|
||||
},
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
@@ -344,6 +504,35 @@ func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string
|
||||
return owner, chat.ID
|
||||
}
|
||||
|
||||
func createQueuedAttachment(
|
||||
t *testing.T,
|
||||
store *postgres.Store,
|
||||
ownerID string,
|
||||
chatID string,
|
||||
) (model.Attachment, model.AttachmentJob) {
|
||||
t.Helper()
|
||||
attachmentID := uuid.NewString()
|
||||
stagingKey := "staging/" + attachmentID + "/original"
|
||||
pending, err := store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: attachmentID, ChatID: chatID, OwnerID: ownerID, Filename: "photo.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64),
|
||||
ObjectKey: stagingKey,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreatePendingAttachment() error = %v", err)
|
||||
}
|
||||
job := model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, Attempt: 1,
|
||||
}
|
||||
queued, err := store.QueueAttachmentWithJobAndOutbox(
|
||||
context.Background(), pending.ID, stagingKey, job, "", "",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("QueueAttachmentWithJobAndOutbox() error = %v", err)
|
||||
}
|
||||
return queued, job
|
||||
}
|
||||
|
||||
func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int {
|
||||
t.Helper()
|
||||
var count int
|
||||
|
||||
@@ -23,6 +23,7 @@ type ObjectStore interface {
|
||||
|
||||
type Config struct {
|
||||
Prefix string
|
||||
OrphanPrefixes []string
|
||||
PageSize int
|
||||
GraceTime time.Duration
|
||||
UnreferencedRetention time.Duration
|
||||
@@ -65,6 +66,11 @@ func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconcile
|
||||
if config.UnreferencedRetention <= 0 {
|
||||
return nil, errors.New("unreferenced retention must be positive")
|
||||
}
|
||||
for _, prefix := range config.OrphanPrefixes {
|
||||
if prefix == "" || prefix == config.Prefix {
|
||||
return nil, errors.New("orphan prefix must be non-empty and distinct")
|
||||
}
|
||||
}
|
||||
if config.Now == nil {
|
||||
config.Now = time.Now
|
||||
}
|
||||
@@ -113,9 +119,35 @@ func (r *Reconciler) Run(ctx context.Context) (Report, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, prefix := range r.config.OrphanPrefixes {
|
||||
if err := r.scanOrphanPrefix(ctx, prefix, anomalyCutoff, &report); err != nil {
|
||||
return report, err
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) scanOrphanPrefix(
|
||||
ctx context.Context,
|
||||
prefix string,
|
||||
cutoff time.Time,
|
||||
report *Report,
|
||||
) error {
|
||||
objects, err := newObjectIterator(ctx, r.objects, prefix, r.config.PageSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for objects.current != nil {
|
||||
if err := r.handleOrphanObject(ctx, *objects.current, cutoff, report); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.advance(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) handlePair(
|
||||
ctx context.Context,
|
||||
metadata model.Attachment,
|
||||
@@ -226,6 +258,14 @@ func (r *Reconciler) handleMissingObject(
|
||||
report *Report,
|
||||
) error {
|
||||
report.MetadataScanned++
|
||||
if metadata.Status == model.AttachmentPending {
|
||||
if metadata.CreatedAt.After(cutoff) {
|
||||
report.FreshPendingAttachments++
|
||||
return nil
|
||||
}
|
||||
report.OldPendingAttachments++
|
||||
return r.deletePair(ctx, metadata, cutoff, report)
|
||||
}
|
||||
if metadata.Status == model.AttachmentDeleting {
|
||||
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||
"attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false),
|
||||
"attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false),
|
||||
"attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false),
|
||||
"staging/pending/original": attachment("pending", "staging/pending/original", model.AttachmentPending, old, false),
|
||||
}}
|
||||
objects := &objectStub{items: map[string]model.StoredObject{
|
||||
"attachments/a/original": {Key: "attachments/a/original", LastModified: old},
|
||||
@@ -28,27 +29,33 @@ func TestReconcilerLifecycleAndRepeat(t *testing.T) {
|
||||
"attachments/c/original": {Key: "attachments/c/original", LastModified: old},
|
||||
"attachments/d/original": {Key: "attachments/d/original", LastModified: fresh},
|
||||
"attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old},
|
||||
"staging/orphan/original": {Key: "staging/orphan/original", LastModified: old},
|
||||
"staging/pending/original": {Key: "staging/pending/original", LastModified: old},
|
||||
}}
|
||||
reconciler, err := New(metadata, objects, Config{
|
||||
Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour,
|
||||
OrphanPrefixes: []string{"staging/"},
|
||||
UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now },
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
report, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, report.OldPendingAttachments)
|
||||
require.Equal(t, 2, report.OldPendingAttachments)
|
||||
require.Equal(t, 1, report.OldUnreferencedAttachments)
|
||||
require.Equal(t, 1, report.FreshUnreferencedAttachments)
|
||||
require.Equal(t, 1, report.DeletedOrphanObjects)
|
||||
require.Equal(t, 2, report.DeletedAttachmentObjects)
|
||||
require.Equal(t, 3, report.DeletedUnreferencedRows)
|
||||
require.Equal(t, 2, report.DeletedOrphanObjects)
|
||||
require.Equal(t, 3, report.DeletedAttachmentObjects)
|
||||
require.Equal(t, 4, report.DeletedUnreferencedRows)
|
||||
require.Contains(t, metadata.items, "attachments/b/original")
|
||||
require.Contains(t, metadata.items, "attachments/d/original")
|
||||
require.NotContains(t, metadata.items, "attachments/a/original")
|
||||
require.NotContains(t, metadata.items, "attachments/c/original")
|
||||
require.NotContains(t, metadata.items, "attachments/missing/original")
|
||||
require.NotContains(t, metadata.items, "staging/pending/original")
|
||||
require.NotContains(t, objects.items, "attachments/orphan/original")
|
||||
require.NotContains(t, objects.items, "staging/orphan/original")
|
||||
require.NotContains(t, objects.items, "staging/pending/original")
|
||||
|
||||
repeated, err := reconciler.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT attachments_status,
|
||||
DROP CONSTRAINT attachments_ready_at;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN error_code TEXT,
|
||||
ADD COLUMN staging_key TEXT;
|
||||
|
||||
UPDATE attachments
|
||||
SET staging_key = object_key
|
||||
WHERE status = 'pending';
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status CHECK (
|
||||
status IN ('pending', 'queued', 'processing', 'ready', 'failed', 'deleting')
|
||||
),
|
||||
ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1),
|
||||
ADD CONSTRAINT attachments_error_code CHECK (
|
||||
(status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR
|
||||
(status <> 'failed' AND error_code IS NULL)
|
||||
),
|
||||
ADD CONSTRAINT attachments_ready_at CHECK (
|
||||
(status = 'ready' AND ready_at IS NOT NULL) OR
|
||||
(status <> 'ready')
|
||||
),
|
||||
ADD CONSTRAINT attachments_staging_key_length CHECK (
|
||||
staging_key IS NULL OR char_length(staging_key) BETWEEN 1 AND 1024
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_status_updated_at_idx
|
||||
ON attachments (status, updated_at, id);
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
lease_token UUID,
|
||||
lease_owner TEXT,
|
||||
lease_until TIMESTAMPTZ,
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1),
|
||||
CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_lease CHECK (
|
||||
(status = 'processing' AND lease_token IS NOT NULL AND lease_owner IS NOT NULL AND lease_until IS NOT NULL) OR
|
||||
(status <> 'processing' AND lease_token IS NULL AND lease_owner IS NULL AND lease_until IS NULL)
|
||||
),
|
||||
CONSTRAINT attachment_jobs_error_code CHECK (
|
||||
error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_jobs_due_idx
|
||||
ON attachment_jobs (available_at, id)
|
||||
WHERE status = 'queued';
|
||||
|
||||
CREATE INDEX attachment_jobs_expired_lease_idx
|
||||
ON attachment_jobs (lease_until, id)
|
||||
WHERE status = 'processing';
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_variants_pk PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT attachment_variants_object_key_unique UNIQUE (object_key),
|
||||
CONSTRAINT attachment_variants_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT attachment_variants_name CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT attachment_variants_content_type CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')),
|
||||
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1),
|
||||
CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachment_variants_dimensions CHECK (width >= 1 AND height >= 1),
|
||||
CONSTRAINT attachment_variants_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024)
|
||||
);
|
||||
|
||||
CREATE TABLE outbox_events (
|
||||
id UUID PRIMARY KEY,
|
||||
topic TEXT NOT NULL,
|
||||
partition_key TEXT NOT NULL,
|
||||
payload JSONB NOT NULL,
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
|
||||
attempt INTEGER NOT NULL,
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
publish_tries INTEGER NOT NULL DEFAULT 0,
|
||||
claim_token UUID,
|
||||
claim_until TIMESTAMPTZ,
|
||||
last_error_code TEXT,
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT outbox_events_job_attempt_unique UNIQUE (job_id, attempt),
|
||||
CONSTRAINT outbox_events_attempt CHECK (attempt >= 1),
|
||||
CONSTRAINT outbox_events_publish_tries CHECK (publish_tries >= 0),
|
||||
CONSTRAINT outbox_events_claim CHECK (
|
||||
(claim_token IS NULL AND claim_until IS NULL) OR
|
||||
(claim_token IS NOT NULL AND claim_until IS NOT NULL)
|
||||
),
|
||||
CONSTRAINT outbox_events_last_error_code CHECK (
|
||||
last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX outbox_events_due_idx
|
||||
ON outbox_events (available_at, id)
|
||||
WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE attachment_replay_audit (
|
||||
id UUID PRIMARY KEY,
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
previous_attempt INTEGER NOT NULL,
|
||||
new_attempt INTEGER NOT NULL,
|
||||
actor TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_replay_attempts CHECK (previous_attempt >= 1 AND new_attempt = previous_attempt + 1),
|
||||
CONSTRAINT attachment_replay_actor_length CHECK (char_length(actor) BETWEEN 1 AND 128),
|
||||
CONSTRAINT attachment_replay_reason_length CHECK (char_length(reason) BETWEEN 1 AND 512)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_replay_audit_job_created_at_idx
|
||||
ON attachment_replay_audit (job_id, created_at);
|
||||
Reference in New Issue
Block a user