forked from mtusi2026/mtusi2026lab5
107 lines
3.3 KiB
Go
107 lines
3.3 KiB
Go
// Package streamqueue adapts durable Valkey Streams to the job queue interface.
|
|
package streamqueue
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/redis/go-redis/v9"
|
|
|
|
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab5/internal/pipeline"
|
|
)
|
|
|
|
const defaultStream = "attachment-jobs"
|
|
const group = "image-workers"
|
|
|
|
// Queue owns a bounded connection pool and one consumer identity.
|
|
type Queue struct {
|
|
client *redis.Client
|
|
consumer string
|
|
cursor string
|
|
stream string
|
|
}
|
|
|
|
// New configures finite broker I/O without opening network connections.
|
|
func New(address, consumer string) (*Queue, error) {
|
|
return NewScoped(address, consumer, defaultStream)
|
|
}
|
|
|
|
// NewScoped isolates an explicitly named queue, including integration-test fixtures.
|
|
func NewScoped(address, consumer, streamName string) (*Queue, error) {
|
|
if address == "" || consumer == "" || streamName == "" {
|
|
return nil, fmt.Errorf("broker address and consumer are required")
|
|
}
|
|
return &Queue{
|
|
client: redis.NewClient(&redis.Options{
|
|
Addr: address,
|
|
PoolSize: 2,
|
|
MaxRetries: 0,
|
|
DialTimeout: 2 * time.Second,
|
|
ReadTimeout: 3 * time.Second,
|
|
WriteTimeout: 2 * time.Second,
|
|
}),
|
|
consumer: consumer,
|
|
cursor: "0-0",
|
|
stream: streamName,
|
|
}, nil
|
|
}
|
|
|
|
// Close releases broker connections after worker shutdown.
|
|
func (q *Queue) Close() error {
|
|
return q.client.Close()
|
|
}
|
|
|
|
// Publish writes a notification; broker AOF appendfsync always is required by deployment.
|
|
func (q *Queue) Publish(ctx context.Context, j pipeline.Job) error {
|
|
data, err := json.Marshal(j)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return q.client.XAdd(ctx, &redis.XAddArgs{Stream: q.stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(data)}}).Err()
|
|
}
|
|
|
|
// Receive reclaims expired deliveries before waiting for one new job.
|
|
func (q *Queue) Receive(ctx context.Context) (*pipeline.Delivery, error) {
|
|
if err := q.client.XGroupCreateMkStream(ctx, q.stream, group, "0").Err(); err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
|
|
return nil, err
|
|
}
|
|
messages, next, err := q.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: q.stream, Group: group, Consumer: q.consumer, MinIdle: 2 * time.Minute, Start: q.cursor, Count: 1}).Result()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
q.cursor = next
|
|
if len(messages) == 0 {
|
|
result, err := q.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: group, Consumer: q.consumer, Streams: []string{q.stream, ">"}, Count: 1, Block: time.Second}).Result()
|
|
if errors.Is(err, redis.Nil) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
messages = result[0].Messages
|
|
}
|
|
if len(messages) == 0 {
|
|
return nil, nil
|
|
}
|
|
message := messages[0]
|
|
raw, ok := message.Values["job"].(string)
|
|
var job pipeline.Job
|
|
if !ok || json.Unmarshal([]byte(raw), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" || job.Version < 1 || job.Attempt < 1 {
|
|
// Malformed notifications contain no trusted intent; SQL outbox remains authoritative.
|
|
return nil, q.Ack(ctx, message.ID)
|
|
}
|
|
return &pipeline.Delivery{Receipt: message.ID, Job: job}, nil
|
|
}
|
|
|
|
// Ack removes a confirmed notification from the pending group and stream.
|
|
func (q *Queue) Ack(ctx context.Context, receipt string) error {
|
|
if err := q.client.XAck(ctx, q.stream, group, receipt).Err(); err != nil {
|
|
return err
|
|
}
|
|
return q.client.XDel(ctx, q.stream, receipt).Err()
|
|
}
|