forked from mtusi2026/mtusi2026lab5
136 lines
3.2 KiB
Go
136 lines
3.2 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
const (
|
|
jobStream = "attachments.jobs"
|
|
jobGroup = "workers"
|
|
jobType = "attachment.process.v1"
|
|
pipelineVersion = 1
|
|
)
|
|
|
|
type jobMessage struct {
|
|
SchemaVersion int `json:"schema_version"`
|
|
JobID string `json:"job_id"`
|
|
Type string `json:"type"`
|
|
AttachmentID string `json:"attachment_id"`
|
|
PipelineVersion int `json:"pipeline_version"`
|
|
Attempt int `json:"attempt"`
|
|
CreatedAt string `json:"created_at"`
|
|
}
|
|
|
|
type Broker struct {
|
|
rdb *redis.Client
|
|
}
|
|
|
|
func openBroker() (*Broker, error) {
|
|
raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
opt, err := redis.ParseURL(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
opt.DialTimeout = 2 * time.Second
|
|
opt.ReadTimeout = 3 * time.Second
|
|
opt.WriteTimeout = 3 * time.Second
|
|
opt.MaxRetries = 0
|
|
return &Broker{rdb: redis.NewClient(opt)}, nil
|
|
}
|
|
|
|
func (b *Broker) ensureGroup(ctx context.Context) error {
|
|
err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err()
|
|
if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *Broker) publish(ctx context.Context, m jobMessage) error {
|
|
body, err := json.Marshal(m)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err()
|
|
}
|
|
|
|
func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
tx, err := db.Begin(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
rows, err := tx.Query(ctx, `
|
|
select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at
|
|
from outbox o
|
|
join jobs j on j.id = o.job_id
|
|
where o.published_at is null and o.available_at <= now()
|
|
order by o.id
|
|
limit 50
|
|
for update of o skip locked`)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var ids []int64
|
|
var msgs []jobMessage
|
|
for rows.Next() {
|
|
var id int64
|
|
var created time.Time
|
|
m := jobMessage{SchemaVersion: 1, Type: jobType}
|
|
if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created); err != nil {
|
|
rows.Close()
|
|
return 0, err
|
|
}
|
|
m.CreatedAt = formatTime(created)
|
|
ids = append(ids, id)
|
|
msgs = append(msgs, m)
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
if len(ids) == 0 {
|
|
return 0, nil
|
|
}
|
|
if err := b.ensureGroup(ctx); err != nil {
|
|
return 0, err
|
|
}
|
|
for _, m := range msgs {
|
|
if err := b.publish(ctx, m); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil {
|
|
return 0, err
|
|
}
|
|
return len(ids), tx.Commit(ctx)
|
|
}
|
|
|
|
func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) {
|
|
for ctx.Err() == nil {
|
|
n, err := relayOnce(ctx, db, b)
|
|
if err != nil {
|
|
log.Warn("outbox relay failed", "err", err.Error())
|
|
time.Sleep(2 * time.Second)
|
|
continue
|
|
}
|
|
if n > 0 {
|
|
log.Info("outbox published", "count", n)
|
|
continue
|
|
}
|
|
time.Sleep(300 * time.Millisecond)
|
|
}
|
|
}
|