Files
mtusi2026lab7/backend/internal/adapter/queue/streams.go

152 lines
4.4 KiB
Go

package queue
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
"time"
)
const stream = "attachment-jobs"
type Broker struct {
client *redis.Client
publisher *redis.Client
store port.PipelineStore
consumer string
}
func New(publisher, client *redis.Client, store port.PipelineStore, consumer string) *Broker {
return &Broker{client: client, publisher: publisher, store: store, consumer: consumer}
}
func (b *Broker) Publish(ctx context.Context) error {
jobs, e := b.store.PendingJobs(ctx)
if e != nil {
return e
}
for _, j := range jobs {
parent := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": j.Traceparent, "tracestate": j.Tracestate})
send, span := otel.Tracer("messenger").Start(parent, "attachment.publish", trace.WithSpanKind(trace.SpanKindProducer))
carrier := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(send, carrier)
j.Traceparent = carrier.Get("traceparent")
j.Tracestate = carrier.Get("tracestate")
j.Attempt++
payload := struct {
Schema int `json:"schema_version"`
Type string `json:"type"`
model.Job
}{1, "attachment.process.v1", j}
raw, e := json.Marshal(payload)
if e != nil {
span.SetStatus(codes.Error, "job_encoding_error")
span.End()
return e
}
if e = b.publisher.XAdd(send, &redis.XAddArgs{Stream: stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(raw)}}).Err(); e != nil {
span.SetStatus(codes.Error, "broker_error")
span.End()
return e
}
if e = b.store.Published(send, j.ID); e != nil {
span.SetStatus(codes.Error, "outbox_persistence_error")
span.End()
return e
}
span.End()
}
return nil
}
func (b *Broker) RunPublisher(ctx context.Context, onError func(error)) {
tick := time.NewTicker(500 * time.Millisecond)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if e := b.Publish(ctx); e != nil {
onError(e)
}
}
}
}
func (b *Broker) ack(ctx context.Context, id string) {
if b.client.XAck(ctx, stream, "workers", id).Err() == nil {
_ = b.client.XDel(ctx, stream, id).Err()
}
}
func (b *Broker) consume(ctx context.Context, m redis.XMessage, process func(context.Context, model.Job) error, onError func(error)) {
raw, ok := m.Values["job"].(string)
if !ok || len(raw) > 2048 {
b.ack(ctx, m.ID)
return
}
var envelope struct {
Schema int `json:"schema_version"`
Type string `json:"type"`
model.Job
}
decoder := json.NewDecoder(bytes.NewBufferString(raw))
decoder.DisallowUnknownFields()
if decoder.Decode(&envelope) != nil || envelope.Schema != 1 || envelope.Type != "attachment.process.v1" || envelope.ID == uuid.Nil || envelope.AttachmentID == uuid.Nil || envelope.PipelineVersion < 1 || envelope.Attempt < 1 {
b.ack(ctx, m.ID)
return
}
work, cancel := context.WithTimeout(ctx, 20*time.Second)
e := process(work, envelope.Job)
cancel()
if e != nil {
onError(e)
} else {
b.ack(ctx, m.ID)
}
}
func (b *Broker) RunWorker(ctx context.Context, process func(context.Context, model.Job) error, onError func(error)) {
for ctx.Err() == nil {
e := b.client.XGroupCreateMkStream(ctx, stream, "workers", "0").Err()
if e != nil && e.Error() != "BUSYGROUP Consumer Group name already exists" {
onError(e)
pause(ctx)
continue
}
// Reclaim abandoned PEL entries; the DB lease and version fence are authoritative.
claimed, _, e := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: stream, Group: "workers", Consumer: b.consumer, MinIdle: 30 * time.Second, Start: "0-0", Count: 1}).Result()
if e == nil && len(claimed) > 0 {
for _, m := range claimed {
b.consume(ctx, m, process, onError)
}
continue
}
rows, e := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: "workers", Consumer: b.consumer, Streams: []string{stream, ">"}, Count: 1, Block: time.Second}).Result()
if errors.Is(e, redis.Nil) {
continue
}
if e != nil {
onError(e)
pause(ctx)
continue
}
for _, row := range rows {
for _, m := range row.Messages {
b.consume(ctx, m, process, onError)
}
}
}
}
func pause(ctx context.Context) {
select {
case <-ctx.Done():
case <-time.After(200 * time.Millisecond):
}
}