forked from mtusi2026/mtusi2026lab6
feat: add bounded observability and verified SLO incident alerts
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
// Package streamqueue adapts durable Valkey Streams to the job queue interface.
|
||||
package streamqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline"
|
||||
)
|
||||
|
||||
const defaultStream = "attachment-jobs"
|
||||
const group = "image-workers"
|
||||
|
||||
// Queue owns a bounded connection pool and one consumer identity.
|
||||
type Queue struct {
|
||||
client *redis.Client
|
||||
consumer string
|
||||
cursor string
|
||||
stream string
|
||||
}
|
||||
|
||||
// New configures finite broker I/O without opening network connections.
|
||||
func New(address, consumer string) (*Queue, error) {
|
||||
return NewScoped(address, consumer, defaultStream)
|
||||
}
|
||||
|
||||
// NewScoped isolates an explicitly named queue, including integration-test fixtures.
|
||||
func NewScoped(address, consumer, streamName string) (*Queue, error) {
|
||||
if address == "" || consumer == "" || streamName == "" {
|
||||
return nil, fmt.Errorf("broker address and consumer are required")
|
||||
}
|
||||
return &Queue{
|
||||
client: redis.NewClient(&redis.Options{
|
||||
Addr: address,
|
||||
PoolSize: 2,
|
||||
MaxRetries: 0,
|
||||
DialTimeout: 2 * time.Second,
|
||||
ReadTimeout: 3 * time.Second,
|
||||
WriteTimeout: 2 * time.Second,
|
||||
}),
|
||||
consumer: consumer,
|
||||
cursor: "0-0",
|
||||
stream: streamName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close releases broker connections after worker shutdown.
|
||||
func (q *Queue) Close() error {
|
||||
return q.client.Close()
|
||||
}
|
||||
|
||||
// Publish writes a notification; broker AOF appendfsync always is required by deployment.
|
||||
func (q *Queue) Publish(ctx context.Context, j pipeline.Job) error {
|
||||
data, err := json.Marshal(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return q.client.XAdd(ctx, &redis.XAddArgs{Stream: q.stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(data)}}).Err()
|
||||
}
|
||||
|
||||
// Receive reclaims expired deliveries before waiting for one new job.
|
||||
func (q *Queue) Receive(ctx context.Context) (*pipeline.Delivery, error) {
|
||||
if err := q.client.XGroupCreateMkStream(ctx, q.stream, group, "0").Err(); err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
|
||||
return nil, err
|
||||
}
|
||||
messages, next, err := q.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: q.stream, Group: group, Consumer: q.consumer, MinIdle: 2 * time.Minute, Start: q.cursor, Count: 1}).Result()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q.cursor = next
|
||||
if len(messages) == 0 {
|
||||
result, err := q.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: group, Consumer: q.consumer, Streams: []string{q.stream, ">"}, Count: 1, Block: time.Second}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
messages = result[0].Messages
|
||||
}
|
||||
if len(messages) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
message := messages[0]
|
||||
raw, ok := message.Values["job"].(string)
|
||||
var job pipeline.Job
|
||||
if !ok || json.Unmarshal([]byte(raw), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" || job.Version < 1 || job.Attempt < 1 {
|
||||
// Malformed notifications contain no trusted intent; SQL outbox remains authoritative.
|
||||
return nil, q.Ack(ctx, message.ID)
|
||||
}
|
||||
return &pipeline.Delivery{Receipt: message.ID, Job: job}, nil
|
||||
}
|
||||
|
||||
// Ack removes a confirmed notification from the pending group and stream.
|
||||
func (q *Queue) Ack(ctx context.Context, receipt string) error {
|
||||
if err := q.client.XAck(ctx, q.stream, group, receipt).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return q.client.XDel(ctx, q.stream, receipt).Err()
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package streamqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline"
|
||||
)
|
||||
|
||||
// queueFixture isolates stream and consumer state while using a real broker.
|
||||
func queueFixture(t *testing.T) *Queue {
|
||||
t.Helper()
|
||||
address := os.Getenv("BROKER_ADDRESS")
|
||||
if address == "" {
|
||||
t.Skip("isolated BROKER_ADDRESS required")
|
||||
}
|
||||
q, err := NewScoped(address, uuid.NewString(), "test-"+uuid.NewString())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := q.client.Del(context.Background(), q.stream).Err(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if err := q.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
return q
|
||||
}
|
||||
|
||||
// TestQueueDeliveryAndAck verifies durable envelope round-trip and acknowledgement.
|
||||
func TestQueueDeliveryAndAck(t *testing.T) {
|
||||
q := queueFixture(t)
|
||||
job := pipeline.Job{SchemaVersion: 1, ID: uuid.New(), AttachmentID: uuid.New(), Version: 1, Attempt: 1, Type: "attachment.process.v1", CreatedAt: time.Now().UTC()}
|
||||
if err := q.Publish(t.Context(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
received, err := q.Receive(t.Context())
|
||||
if err != nil || received == nil || received.Job.ID != job.ID {
|
||||
t.Fatalf("receive: %v", err)
|
||||
}
|
||||
if err := q.Ack(t.Context(), received.Receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if delivery, err := q.Receive(t.Context()); err != nil || delivery != nil {
|
||||
t.Fatal("acked job redelivered")
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueMalformedNotification verifies malformed data cannot become a processing job.
|
||||
func TestQueueMalformedNotification(t *testing.T) {
|
||||
q := queueFixture(t)
|
||||
if err := q.client.Do(t.Context(), "XADD", q.stream, "*", "job", "invalid-json").Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if delivery, err := q.Receive(t.Context()); err != nil || delivery != nil {
|
||||
t.Fatal("invalid job delivered")
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueReclaimsAbandonedDelivery verifies a consumer crash does not lose a pending notification.
|
||||
func TestQueueReclaimsAbandonedDelivery(t *testing.T) {
|
||||
q := queueFixture(t)
|
||||
job := pipeline.Job{SchemaVersion: 1, ID: uuid.New(), AttachmentID: uuid.New(), Version: 1, Attempt: 1, Type: "attachment.process.v1", CreatedAt: time.Now().UTC()}
|
||||
if err := q.Publish(t.Context(), job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
received, err := q.Receive(t.Context())
|
||||
if err != nil || received == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.client.Do(t.Context(), "XCLAIM", q.stream, group, "dead-consumer", 0, received.Receipt, "IDLE", 180000).Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reclaimed, err := q.Receive(t.Context())
|
||||
if err != nil || reclaimed == nil || reclaimed.Receipt != received.Receipt {
|
||||
t.Fatal("pending delivery lost")
|
||||
}
|
||||
if err := q.Ack(t.Context(), reclaimed.Receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueCanceledOperations checks broker failures stay visible to the durable coordinator.
|
||||
func TestQueueCanceledOperations(t *testing.T) {
|
||||
q := queueFixture(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
t.Run("publish", func(t *testing.T) {
|
||||
if err := q.Publish(ctx, pipeline.Job{}); err == nil {
|
||||
t.Fatal("publish succeeded")
|
||||
}
|
||||
})
|
||||
t.Run("receive", func(t *testing.T) {
|
||||
if _, err := q.Receive(ctx); err == nil {
|
||||
t.Fatal("receive succeeded")
|
||||
}
|
||||
})
|
||||
t.Run("ack", func(t *testing.T) {
|
||||
if err := q.Ack(ctx, "0-0"); err == nil {
|
||||
t.Fatal("ack succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestQueueConfiguration rejects absent dependencies without opening sockets.
|
||||
func TestQueueConfiguration(t *testing.T) {
|
||||
if _, err := New("", ""); err == nil {
|
||||
t.Fatal("empty configuration accepted")
|
||||
}
|
||||
q, err := New("127.0.0.1:1", "consumer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueRejectsUnserializableTimestamp prevents sending an envelope outside JSON time bounds.
|
||||
func TestQueueRejectsUnserializableTimestamp(t *testing.T) {
|
||||
q := queueFixture(t)
|
||||
if err := q.Publish(t.Context(), pipeline.Job{CreatedAt: time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC)}); err == nil {
|
||||
t.Fatal("invalid timestamp published")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user