package main import ( "context" "encoding/json" "log/slog" "strings" "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" ) const ( jobStream = "attachments.jobs" jobGroup = "workers" jobType = "attachment.process.v1" pipelineVersion = 1 ) type jobMessage struct { SchemaVersion int `json:"schema_version"` JobID string `json:"job_id"` Type string `json:"type"` AttachmentID string `json:"attachment_id"` PipelineVersion int `json:"pipeline_version"` Attempt int `json:"attempt"` CreatedAt string `json:"created_at"` } type Broker struct { rdb *redis.Client } func openBroker() (*Broker, error) { raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE") if err != nil { return nil, err } opt, err := redis.ParseURL(raw) if err != nil { return nil, err } opt.DialTimeout = 2 * time.Second opt.ReadTimeout = 3 * time.Second opt.WriteTimeout = 3 * time.Second opt.MaxRetries = 0 return &Broker{rdb: redis.NewClient(opt)}, nil } func (b *Broker) ensureGroup(ctx context.Context) error { err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err() if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { return err } return nil } func (b *Broker) publish(ctx context.Context, m jobMessage) error { body, err := json.Marshal(m) if err != nil { return err } return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err() } func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() tx, err := db.Begin(ctx) if err != nil { return 0, err } defer tx.Rollback(ctx) rows, err := tx.Query(ctx, ` select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at from outbox o join jobs j on j.id = o.job_id where o.published_at is null and o.available_at <= now() order by o.id limit 50 for update of o skip locked`) if err != nil { return 0, err } var ids []int64 var msgs []jobMessage for rows.Next() { var id int64 var created time.Time m := jobMessage{SchemaVersion: 1, Type: jobType} if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created); err != nil { rows.Close() return 0, err } m.CreatedAt = formatTime(created) ids = append(ids, id) msgs = append(msgs, m) } rows.Close() if err := rows.Err(); err != nil { return 0, err } if len(ids) == 0 { return 0, nil } if err := b.ensureGroup(ctx); err != nil { return 0, err } for _, m := range msgs { if err := b.publish(ctx, m); err != nil { return 0, err } } if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil { return 0, err } return len(ids), tx.Commit(ctx) } func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) { for ctx.Err() == nil { n, err := relayOnce(ctx, db, b) if err != nil { log.Warn("outbox relay failed", "err", err.Error()) time.Sleep(2 * time.Second) continue } if n > 0 { log.Info("outbox published", "count", n) continue } time.Sleep(300 * time.Millisecond) } }