Files
mtusi2026lab5/backend/internal/adapter/queue/streams.go

134 lines
3.5 KiB
Go

package queue
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"time"
)
const stream = "attachment-jobs"
type Broker struct {
client *redis.Client
store port.PipelineStore
consumer string
}
func New(client *redis.Client, store port.PipelineStore, consumer string) *Broker {
return &Broker{client, store, consumer}
}
func (b *Broker) Publish(ctx context.Context) error {
jobs, e := b.store.PendingJobs(ctx)
if e != nil {
return e
}
for _, j := range jobs {
j.Attempt++
payload := struct {
Schema int `json:"schema_version"`
Type string `json:"type"`
model.Job
}{1, "attachment.process.v1", j}
raw, e := json.Marshal(payload)
if e != nil {
return e
}
if e = b.client.XAdd(ctx, &redis.XAddArgs{Stream: stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(raw)}}).Err(); e != nil {
return e
}
if e = b.store.Published(ctx, j.ID); e != nil {
return e
}
}
return nil
}
func (b *Broker) RunPublisher(ctx context.Context, onError func(error)) {
tick := time.NewTicker(500 * time.Millisecond)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if e := b.Publish(ctx); e != nil {
onError(e)
}
}
}
}
func (b *Broker) ack(ctx context.Context, id string) {
if b.client.XAck(ctx, stream, "workers", id).Err() == nil {
_ = b.client.XDel(ctx, stream, id).Err()
}
}
func (b *Broker) consume(ctx context.Context, m redis.XMessage, process func(context.Context, model.Job) error, onError func(error)) {
raw, ok := m.Values["job"].(string)
if !ok || len(raw) > 2048 {
b.ack(ctx, m.ID)
return
}
var envelope struct {
Schema int `json:"schema_version"`
Type string `json:"type"`
model.Job
}
decoder := json.NewDecoder(bytes.NewBufferString(raw))
decoder.DisallowUnknownFields()
if decoder.Decode(&envelope) != nil || envelope.Schema != 1 || envelope.Type != "attachment.process.v1" || envelope.ID == uuid.Nil || envelope.AttachmentID == uuid.Nil || envelope.PipelineVersion < 1 || envelope.Attempt < 1 {
b.ack(ctx, m.ID)
return
}
work, cancel := context.WithTimeout(ctx, 20*time.Second)
e := process(work, envelope.Job)
cancel()
if e != nil {
onError(e)
} else {
b.ack(ctx, m.ID)
}
}
func (b *Broker) RunWorker(ctx context.Context, process func(context.Context, model.Job) error, onError func(error)) {
for ctx.Err() == nil {
e := b.client.XGroupCreateMkStream(ctx, stream, "workers", "0").Err()
if e != nil && e.Error() != "BUSYGROUP Consumer Group name already exists" {
onError(e)
pause(ctx)
continue
}
// Reclaim abandoned PEL entries; the DB lease and version fence are authoritative.
claimed, _, e := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: stream, Group: "workers", Consumer: b.consumer, MinIdle: 30 * time.Second, Start: "0-0", Count: 1}).Result()
if e == nil && len(claimed) > 0 {
for _, m := range claimed {
b.consume(ctx, m, process, onError)
}
continue
}
rows, e := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: "workers", Consumer: b.consumer, Streams: []string{stream, ">"}, Count: 1, Block: time.Second}).Result()
if errors.Is(e, redis.Nil) {
continue
}
if e != nil {
onError(e)
pause(ctx)
continue
}
for _, row := range rows {
for _, m := range row.Messages {
b.consume(ctx, m, process, onError)
}
}
}
}
func pause(ctx context.Context) {
select {
case <-ctx.Done():
case <-time.After(200 * time.Millisecond):
}
}