package main import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log/slog" "math/rand/v2" "os" "os/signal" "strings" "sync" "syscall" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" ) var errSource = errors.New("staging object missing or corrupted") type Worker struct { db *pgxpool.Pool store *Store broker *Broker log *slog.Logger owner string timeout time.Duration lease time.Duration backoffBase time.Duration backoffMax time.Duration maxBytes int64 maxPixels int64 fault string } type claimedJob struct { id, attachmentID, owner string attempt int } type source struct { stagingKey, contentType, sha256 string size int64 } func finalKey(attachmentID string) string { return objectPrefix + attachmentID + "/original" } func variantKey(attachmentID string, spec variantSpec) string { return fmt.Sprintf("%s%s/v%d/%s.%s", objectPrefix, attachmentID, pipelineVersion, spec.name, spec.ext) } func (w *Worker) inject(point string) { if w.fault == point { w.log.Error("fault injected", "point", point) os.Exit(3) } } func (w *Worker) backoff(attempt int) time.Duration { d := w.backoffBase << (attempt - 1) if d <= 0 || d > w.backoffMax { d = w.backoffMax } return d/2 + rand.N(d/2+1) } func (w *Worker) claim(ctx context.Context, jobID string) (*claimedJob, error) { tx, err := w.db.Begin(ctx) if err != nil { return nil, err } defer tx.Rollback(ctx) job := &claimedJob{id: jobID, owner: w.owner} err = tx.QueryRow(ctx, ` update jobs set status = 'processing', lease_owner = $2, lease_until = now() + make_interval(secs => $3), updated_at = now() where id = $1::uuid and status = 'queued' and next_attempt_at <= now() returning attachment_id::text, attempt`, jobID, w.owner, w.lease.Seconds()).Scan(&job.attachmentID, &job.attempt) if errors.Is(err, pgx.ErrNoRows) { return nil, nil } if err != nil { return nil, err } if _, err := tx.Exec(ctx, `update attachments set status = 'processing', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { return nil, err } return job, tx.Commit(ctx) } func (w *Worker) fetch(ctx context.Context, src source) ([]byte, error) { obj, info, err := w.store.Get(ctx, src.stagingKey) if err != nil { if isNoSuchKey(err) { return nil, errSource } return nil, err } defer obj.Close() if info.Size != src.size || info.Size > w.maxBytes { return nil, errSource } data, err := io.ReadAll(io.LimitReader(obj, w.maxBytes+1)) if err != nil { return nil, err } sum := sha256.Sum256(data) if int64(len(data)) != src.size || hex.EncodeToString(sum[:]) != src.sha256 { return nil, errSource } return data, nil } func (w *Worker) execute(ctx context.Context, job *claimedJob) error { ctx, cancel := context.WithTimeout(ctx, w.timeout) defer cancel() var src source err := w.db.QueryRow(ctx, ` select coalesce(staging_key, ''), content_type, sha256, size_bytes from attachments where id = $1::uuid`, job.attachmentID).Scan(&src.stagingKey, &src.contentType, &src.sha256, &src.size) if err != nil { return err } data, err := w.fetch(ctx, src) if err != nil { return err } variants, err := renderVariants(ctx, data, w.maxPixels) if err != nil { return err } original := finalKey(job.attachmentID) if err := w.store.Put(ctx, original, bytes.NewReader(data), int64(len(data)), src.contentType); err != nil { return err } for _, v := range variants { if err := w.store.Put(ctx, variantKey(job.attachmentID, v.spec), bytes.NewReader(v.data), int64(len(v.data)), v.spec.contentType); err != nil { return err } } w.inject("after_store") tx, err := w.db.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) tag, err := tx.Exec(ctx, ` update jobs set status = 'ready', lease_owner = null, lease_until = null, last_error = null, updated_at = now() where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`, job.id, job.owner, job.attempt) if err != nil { return err } if tag.RowsAffected() == 0 { w.log.Warn("lease lost, result discarded", "job_id", job.id, "attempt", job.attempt) return nil } for _, v := range variants { _, err := tx.Exec(ctx, ` insert into attachment_variants(attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) values($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9) on conflict (attachment_id, pipeline_version, name) do update set object_key = excluded.object_key, content_type = excluded.content_type, size_bytes = excluded.size_bytes, width = excluded.width, height = excluded.height, sha256 = excluded.sha256`, job.attachmentID, pipelineVersion, v.spec.name, variantKey(job.attachmentID, v.spec), v.spec.contentType, len(v.data), v.width, v.height, v.sha256) if err != nil { return err } } if _, err := tx.Exec(ctx, ` update attachments set status = 'ready', error_code = null, object_key = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, original); err != nil { return err } if err := tx.Commit(ctx); err != nil { return err } w.log.Info("job ready", "job_id", job.id, "attachment_id", job.attachmentID, "attempt", job.attempt) w.inject("before_ack") if err := w.store.Remove(ctx, src.stagingKey); err != nil { w.log.Warn("staging cleanup deferred to janitor", "attachment_id", job.attachmentID, "err", err.Error()) } return nil } func (w *Worker) settle(ctx context.Context, job *claimedJob, permanent bool, code, reason string) error { tx, err := w.db.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) var maxAttempts int if err := tx.QueryRow(ctx, `select max_attempts from jobs where id = $1::uuid`, job.id).Scan(&maxAttempts); err != nil { return err } fence := ` where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3` if !permanent && job.attempt < maxAttempts { delay := w.backoff(job.attempt) tag, err := tx.Exec(ctx, ` update jobs set status = 'queued', attempt = attempt + 1, next_attempt_at = now() + make_interval(secs => $4), lease_owner = null, lease_until = null, last_error = $5, updated_at = now()`+fence, job.id, job.owner, job.attempt, delay.Seconds(), reason) if err != nil || tag.RowsAffected() == 0 { return err } if _, err := tx.Exec(ctx, ` insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now() + make_interval(secs => $3), now())`, job.id, job.attempt+1, delay.Seconds()); err != nil { return err } if _, err := tx.Exec(ctx, `update attachments set status = 'queued', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { return err } w.log.Warn("job retry scheduled", "job_id", job.id, "attempt", job.attempt, "next_attempt", job.attempt+1, "delay_ms", delay.Milliseconds(), "reason", reason) return tx.Commit(ctx) } if !permanent { code = "retry_exhausted" } tag, err := tx.Exec(ctx, ` update jobs set status = 'failed', lease_owner = null, lease_until = null, last_error = $4, updated_at = now()`+fence, job.id, job.owner, job.attempt, reason) if err != nil || tag.RowsAffected() == 0 { return err } if _, err := tx.Exec(ctx, ` update attachments set status = 'failed', error_code = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, code); err != nil { return err } w.log.Error("job failed", "job_id", job.id, "attempt", job.attempt, "error_code", code, "reason", reason) return tx.Commit(ctx) } func (w *Worker) handle(ctx context.Context, msg redis.XMessage) bool { var m jobMessage raw, _ := msg.Values["job"].(string) if err := json.Unmarshal([]byte(raw), &m); err != nil || m.SchemaVersion != 1 || m.Type != jobType || !isUUID(m.JobID) { w.log.Warn("invalid job message dropped", "stream_id", msg.ID) return true } job, err := w.claim(ctx, m.JobID) if err != nil { w.log.Warn("claim failed", "job_id", m.JobID, "err", err.Error()) return false } if job == nil { var status string var attempt int _ = w.db.QueryRow(ctx, `select status, attempt from jobs where id = $1::uuid`, m.JobID).Scan(&status, &attempt) w.log.Info("job delivery skipped", "job_id", m.JobID, "message_attempt", m.Attempt, "status", status, "attempt", attempt) return true } err = w.execute(ctx, job) if err == nil { return true } permanent, code := false, "processing_failed" switch { case errors.Is(err, errInvalidImage): permanent, code = true, "invalid_image" case errors.Is(err, errSource): permanent = true } if err := w.settle(ctx, job, permanent, code, err.Error()); err != nil { w.log.Warn("settle failed", "job_id", job.id, "err", err.Error()) return false } return true } func (w *Worker) read(ctx context.Context, consumer, id string, block time.Duration) ([]redis.XMessage, error) { res, err := w.broker.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: jobGroup, Consumer: consumer, Streams: []string{jobStream, id}, Count: 1, Block: block, }).Result() if len(res) == 0 { return nil, err } return res[0].Messages, err } func (w *Worker) consume(ctx context.Context, consumer string) { pending := true nextClaim := time.Now().Add(w.lease) for ctx.Err() == nil { var msgs []redis.XMessage var err error switch { case pending: msgs, err = w.read(ctx, consumer, "0", -1) pending = len(msgs) > 0 case time.Now().After(nextClaim): msgs, _, err = w.broker.rdb.XAutoClaim(ctx, &redis.XAutoClaimArgs{ Stream: jobStream, Group: jobGroup, Consumer: consumer, MinIdle: w.lease, Start: "0-0", Count: 1, }).Result() nextClaim = time.Now().Add(15 * time.Second) default: msgs, err = w.read(ctx, consumer, ">", 5*time.Second) } if err != nil && !errors.Is(err, redis.Nil) { if ctx.Err() != nil { return } if strings.Contains(err.Error(), "NOGROUP") { _ = w.broker.ensureGroup(ctx) } w.log.Warn("broker read failed", "consumer", consumer, "err", err.Error()) time.Sleep(2 * time.Second) continue } for _, msg := range msgs { if !w.handle(context.WithoutCancel(ctx), msg) { pending = false continue } if err := w.broker.rdb.XAck(context.WithoutCancel(ctx), jobStream, jobGroup, msg.ID).Err(); err != nil { w.log.Warn("ack failed", "stream_id", msg.ID, "err", err.Error()) } } } } func (w *Worker) reap(ctx context.Context) error { rows, err := w.db.Query(ctx, ` select id::text, attachment_id::text, attempt, coalesce(lease_owner, '') from jobs where status = 'processing' and lease_until < now() limit 50`) if err != nil { return err } jobs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*claimedJob, error) { j := &claimedJob{} return j, row.Scan(&j.id, &j.attachmentID, &j.attempt, &j.owner) }) if err != nil { return err } for _, j := range jobs { w.log.Warn("lease expired", "job_id", j.id, "attempt", j.attempt) if err := w.settle(ctx, j, false, "", "lease expired"); err != nil { return err } } return nil } func runWorker(db *pgxpool.Pool, store *Store, broker *Broker, log *slog.Logger, w *Worker, concurrency int) { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) defer stop() for { err := broker.ensureGroup(ctx) if err == nil { break } log.Warn("broker not ready", "err", err.Error()) select { case <-ctx.Done(): return case <-time.After(2 * time.Second): } } host, _ := os.Hostname() var wg sync.WaitGroup wg.Go(func() { for ctx.Err() == nil { if err := w.reap(ctx); err != nil && ctx.Err() == nil { log.Warn("lease reaper failed", "err", err.Error()) } select { case <-ctx.Done(): case <-time.After(5 * time.Second): } } }) for i := range concurrency { consumer := fmt.Sprintf("%s-%d", host, i) wg.Go(func() { w.consume(ctx, consumer) }) } log.Info("worker started", "owner", w.owner, "concurrency", concurrency, "lease_s", w.lease.Seconds()) <-ctx.Done() log.Info("worker stopping") wg.Wait() } func replayJob(ctx context.Context, db *pgxpool.Pool, attachmentID string, maxAttempts int) (string, error) { tx, err := db.Begin(ctx) if err != nil { return "", err } defer tx.Rollback(ctx) var jobID, status string var attempt int var lastErr *string err = tx.QueryRow(ctx, `select id::text, status, attempt, last_error from jobs where attachment_id = $1::uuid for update`, attachmentID).Scan(&jobID, &status, &attempt, &lastErr) if errors.Is(err, pgx.ErrNoRows) { return "not_found", nil } if err != nil { return "", err } if status != "failed" { return "skipped_" + status, nil } if _, err := tx.Exec(ctx, `insert into job_replays(job_id, previous_error, replayed_at) values($1::uuid, $2, now())`, jobID, lastErr); err != nil { return "", err } if _, err := tx.Exec(ctx, ` update jobs set status = 'queued', attempt = attempt + 1, max_attempts = attempt + $2, next_attempt_at = now(), last_error = null, updated_at = now() where id = $1::uuid`, jobID, maxAttempts); err != nil { return "", err } if _, err := tx.Exec(ctx, `insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now(), now())`, jobID, attempt+1); err != nil { return "", err } if _, err := tx.Exec(ctx, `update attachments set status = 'queued', error_code = null, updated_at = now() where id = $1::uuid`, attachmentID); err != nil { return "", err } return "queued", tx.Commit(ctx) } func listFailed(ctx context.Context, db *pgxpool.Pool) ([]map[string]any, error) { rows, err := db.Query(ctx, ` select j.attachment_id::text, j.id::text, j.attempt, coalesce(a.error_code, ''), coalesce(j.last_error, ''), j.updated_at from jobs j join attachments a on a.id = j.attachment_id where j.status = 'failed' order by j.updated_at`) if err != nil { return nil, err } return pgx.CollectRows(rows, func(row pgx.CollectableRow) (map[string]any, error) { var att, job, code, reason string var attempt int var updated time.Time err := row.Scan(&att, &job, &attempt, &code, &reason, &updated) return map[string]any{"attachment_id": att, "job_id": job, "attempt": attempt, "error_code": code, "last_error": reason, "failed_at": formatTime(updated)}, err }) }