feat: add bounded observability and verified SLO incident alerts

This commit is contained in:
Vladislav Agapov
2026-10-29 18:01:11 +03:00
parent 470d9d6f1e
commit 6c7601802d
325 changed files with 24988 additions and 44 deletions
+106
View File
@@ -0,0 +1,106 @@
// Package streamqueue adapts durable Valkey Streams to the job queue interface.
package streamqueue
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline"
)
const defaultStream = "attachment-jobs"
const group = "image-workers"
// Queue owns a bounded connection pool and one consumer identity.
type Queue struct {
client *redis.Client
consumer string
cursor string
stream string
}
// New configures finite broker I/O without opening network connections.
func New(address, consumer string) (*Queue, error) {
return NewScoped(address, consumer, defaultStream)
}
// NewScoped isolates an explicitly named queue, including integration-test fixtures.
func NewScoped(address, consumer, streamName string) (*Queue, error) {
if address == "" || consumer == "" || streamName == "" {
return nil, fmt.Errorf("broker address and consumer are required")
}
return &Queue{
client: redis.NewClient(&redis.Options{
Addr: address,
PoolSize: 2,
MaxRetries: 0,
DialTimeout: 2 * time.Second,
ReadTimeout: 3 * time.Second,
WriteTimeout: 2 * time.Second,
}),
consumer: consumer,
cursor: "0-0",
stream: streamName,
}, nil
}
// Close releases broker connections after worker shutdown.
func (q *Queue) Close() error {
return q.client.Close()
}
// Publish writes a notification; broker AOF appendfsync always is required by deployment.
func (q *Queue) Publish(ctx context.Context, j pipeline.Job) error {
data, err := json.Marshal(j)
if err != nil {
return err
}
return q.client.XAdd(ctx, &redis.XAddArgs{Stream: q.stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(data)}}).Err()
}
// Receive reclaims expired deliveries before waiting for one new job.
func (q *Queue) Receive(ctx context.Context) (*pipeline.Delivery, error) {
if err := q.client.XGroupCreateMkStream(ctx, q.stream, group, "0").Err(); err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
return nil, err
}
messages, next, err := q.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: q.stream, Group: group, Consumer: q.consumer, MinIdle: 2 * time.Minute, Start: q.cursor, Count: 1}).Result()
if err != nil {
return nil, err
}
q.cursor = next
if len(messages) == 0 {
result, err := q.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: group, Consumer: q.consumer, Streams: []string{q.stream, ">"}, Count: 1, Block: time.Second}).Result()
if errors.Is(err, redis.Nil) {
return nil, nil
}
if err != nil {
return nil, err
}
messages = result[0].Messages
}
if len(messages) == 0 {
return nil, nil
}
message := messages[0]
raw, ok := message.Values["job"].(string)
var job pipeline.Job
if !ok || json.Unmarshal([]byte(raw), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" || job.Version < 1 || job.Attempt < 1 {
// Malformed notifications contain no trusted intent; SQL outbox remains authoritative.
return nil, q.Ack(ctx, message.ID)
}
return &pipeline.Delivery{Receipt: message.ID, Job: job}, nil
}
// Ack removes a confirmed notification from the pending group and stream.
func (q *Queue) Ack(ctx context.Context, receipt string) error {
if err := q.client.XAck(ctx, q.stream, group, receipt).Err(); err != nil {
return err
}
return q.client.XDel(ctx, q.stream, receipt).Err()
}
+131
View File
@@ -0,0 +1,131 @@
package streamqueue
import (
"context"
"os"
"testing"
"time"
"github.com/google/uuid"
"gitea.privcloud.tech/vlad_marlo/mtusi2026lab6/internal/pipeline"
)
// queueFixture isolates stream and consumer state while using a real broker.
func queueFixture(t *testing.T) *Queue {
t.Helper()
address := os.Getenv("BROKER_ADDRESS")
if address == "" {
t.Skip("isolated BROKER_ADDRESS required")
}
q, err := NewScoped(address, uuid.NewString(), "test-"+uuid.NewString())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := q.client.Del(context.Background(), q.stream).Err(); err != nil {
t.Error(err)
}
if err := q.Close(); err != nil {
t.Error(err)
}
})
return q
}
// TestQueueDeliveryAndAck verifies durable envelope round-trip and acknowledgement.
func TestQueueDeliveryAndAck(t *testing.T) {
q := queueFixture(t)
job := pipeline.Job{SchemaVersion: 1, ID: uuid.New(), AttachmentID: uuid.New(), Version: 1, Attempt: 1, Type: "attachment.process.v1", CreatedAt: time.Now().UTC()}
if err := q.Publish(t.Context(), job); err != nil {
t.Fatal(err)
}
received, err := q.Receive(t.Context())
if err != nil || received == nil || received.Job.ID != job.ID {
t.Fatalf("receive: %v", err)
}
if err := q.Ack(t.Context(), received.Receipt); err != nil {
t.Fatal(err)
}
if delivery, err := q.Receive(t.Context()); err != nil || delivery != nil {
t.Fatal("acked job redelivered")
}
}
// TestQueueMalformedNotification verifies malformed data cannot become a processing job.
func TestQueueMalformedNotification(t *testing.T) {
q := queueFixture(t)
if err := q.client.Do(t.Context(), "XADD", q.stream, "*", "job", "invalid-json").Err(); err != nil {
t.Fatal(err)
}
if delivery, err := q.Receive(t.Context()); err != nil || delivery != nil {
t.Fatal("invalid job delivered")
}
}
// TestQueueReclaimsAbandonedDelivery verifies a consumer crash does not lose a pending notification.
func TestQueueReclaimsAbandonedDelivery(t *testing.T) {
q := queueFixture(t)
job := pipeline.Job{SchemaVersion: 1, ID: uuid.New(), AttachmentID: uuid.New(), Version: 1, Attempt: 1, Type: "attachment.process.v1", CreatedAt: time.Now().UTC()}
if err := q.Publish(t.Context(), job); err != nil {
t.Fatal(err)
}
received, err := q.Receive(t.Context())
if err != nil || received == nil {
t.Fatal(err)
}
if err := q.client.Do(t.Context(), "XCLAIM", q.stream, group, "dead-consumer", 0, received.Receipt, "IDLE", 180000).Err(); err != nil {
t.Fatal(err)
}
reclaimed, err := q.Receive(t.Context())
if err != nil || reclaimed == nil || reclaimed.Receipt != received.Receipt {
t.Fatal("pending delivery lost")
}
if err := q.Ack(t.Context(), reclaimed.Receipt); err != nil {
t.Fatal(err)
}
}
// TestQueueCanceledOperations checks broker failures stay visible to the durable coordinator.
func TestQueueCanceledOperations(t *testing.T) {
q := queueFixture(t)
ctx, cancel := context.WithCancel(t.Context())
cancel()
t.Run("publish", func(t *testing.T) {
if err := q.Publish(ctx, pipeline.Job{}); err == nil {
t.Fatal("publish succeeded")
}
})
t.Run("receive", func(t *testing.T) {
if _, err := q.Receive(ctx); err == nil {
t.Fatal("receive succeeded")
}
})
t.Run("ack", func(t *testing.T) {
if err := q.Ack(ctx, "0-0"); err == nil {
t.Fatal("ack succeeded")
}
})
}
// TestQueueConfiguration rejects absent dependencies without opening sockets.
func TestQueueConfiguration(t *testing.T) {
if _, err := New("", ""); err == nil {
t.Fatal("empty configuration accepted")
}
q, err := New("127.0.0.1:1", "consumer")
if err != nil {
t.Fatal(err)
}
if err := q.Close(); err != nil {
t.Fatal(err)
}
}
// TestQueueRejectsUnserializableTimestamp prevents sending an envelope outside JSON time bounds.
func TestQueueRejectsUnserializableTimestamp(t *testing.T) {
q := queueFixture(t)
if err := q.Publish(t.Context(), pipeline.Job{CreatedAt: time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC)}); err == nil {
t.Fatal("invalid timestamp published")
}
}