Files
mtusi2026lab5-fork/broker.go
T

136 lines
3.2 KiB
Go

package main
import (
"context"
"encoding/json"
"log/slog"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
const (
jobStream = "attachments.jobs"
jobGroup = "workers"
jobType = "attachment.process.v1"
pipelineVersion = 1
)
type jobMessage struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
}
type Broker struct {
rdb *redis.Client
}
func openBroker() (*Broker, error) {
raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE")
if err != nil {
return nil, err
}
opt, err := redis.ParseURL(raw)
if err != nil {
return nil, err
}
opt.DialTimeout = 2 * time.Second
opt.ReadTimeout = 3 * time.Second
opt.WriteTimeout = 3 * time.Second
opt.MaxRetries = 0
return &Broker{rdb: redis.NewClient(opt)}, nil
}
func (b *Broker) ensureGroup(ctx context.Context) error {
err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err()
if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
return err
}
return nil
}
func (b *Broker) publish(ctx context.Context, m jobMessage) error {
body, err := json.Marshal(m)
if err != nil {
return err
}
return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err()
}
func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
tx, err := db.Begin(ctx)
if err != nil {
return 0, err
}
defer tx.Rollback(ctx)
rows, err := tx.Query(ctx, `
select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at
from outbox o
join jobs j on j.id = o.job_id
where o.published_at is null and o.available_at <= now()
order by o.id
limit 50
for update of o skip locked`)
if err != nil {
return 0, err
}
var ids []int64
var msgs []jobMessage
for rows.Next() {
var id int64
var created time.Time
m := jobMessage{SchemaVersion: 1, Type: jobType}
if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created); err != nil {
rows.Close()
return 0, err
}
m.CreatedAt = formatTime(created)
ids = append(ids, id)
msgs = append(msgs, m)
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
if len(ids) == 0 {
return 0, nil
}
if err := b.ensureGroup(ctx); err != nil {
return 0, err
}
for _, m := range msgs {
if err := b.publish(ctx, m); err != nil {
return 0, err
}
}
if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil {
return 0, err
}
return len(ids), tx.Commit(ctx)
}
func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) {
for ctx.Err() == nil {
n, err := relayOnce(ctx, db, b)
if err != nil {
log.Warn("outbox relay failed", "err", err.Error())
time.Sleep(2 * time.Second)
continue
}
if n > 0 {
log.Info("outbox published", "count", n)
continue
}
time.Sleep(300 * time.Millisecond)
}
}