Files
mtusi2026lab5/worker.go
T

449 lines
14 KiB
Go

package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
var errSource = errors.New("staging object missing or corrupted")
type Worker struct {
db *pgxpool.Pool
store *Store
broker *Broker
log *slog.Logger
owner string
timeout time.Duration
lease time.Duration
backoffBase time.Duration
backoffMax time.Duration
maxBytes int64
maxPixels int64
fault string
}
type claimedJob struct {
id, attachmentID, owner string
attempt int
}
type source struct {
stagingKey, contentType, sha256 string
size int64
}
func finalKey(attachmentID string) string {
return objectPrefix + attachmentID + "/original"
}
func variantKey(attachmentID string, spec variantSpec) string {
return fmt.Sprintf("%s%s/v%d/%s.%s", objectPrefix, attachmentID, pipelineVersion, spec.name, spec.ext)
}
func (w *Worker) inject(point string) {
if w.fault == point {
w.log.Error("fault injected", "point", point)
os.Exit(3)
}
}
func (w *Worker) backoff(attempt int) time.Duration {
d := w.backoffBase << (attempt - 1)
if d <= 0 || d > w.backoffMax {
d = w.backoffMax
}
return d/2 + rand.N(d/2+1)
}
func (w *Worker) claim(ctx context.Context, jobID string) (*claimedJob, error) {
tx, err := w.db.Begin(ctx)
if err != nil {
return nil, err
}
defer tx.Rollback(ctx)
job := &claimedJob{id: jobID, owner: w.owner}
err = tx.QueryRow(ctx, `
update jobs set status = 'processing', lease_owner = $2,
lease_until = now() + make_interval(secs => $3), updated_at = now()
where id = $1::uuid and status = 'queued' and next_attempt_at <= now()
returning attachment_id::text, attempt`, jobID, w.owner, w.lease.Seconds()).Scan(&job.attachmentID, &job.attempt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
if _, err := tx.Exec(ctx, `update attachments set status = 'processing', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil {
return nil, err
}
return job, tx.Commit(ctx)
}
func (w *Worker) fetch(ctx context.Context, src source) ([]byte, error) {
obj, info, err := w.store.Get(ctx, src.stagingKey)
if err != nil {
if isNoSuchKey(err) {
return nil, errSource
}
return nil, err
}
defer obj.Close()
if info.Size != src.size || info.Size > w.maxBytes {
return nil, errSource
}
data, err := io.ReadAll(io.LimitReader(obj, w.maxBytes+1))
if err != nil {
return nil, err
}
sum := sha256.Sum256(data)
if int64(len(data)) != src.size || hex.EncodeToString(sum[:]) != src.sha256 {
return nil, errSource
}
return data, nil
}
func (w *Worker) execute(ctx context.Context, job *claimedJob) error {
ctx, cancel := context.WithTimeout(ctx, w.timeout)
defer cancel()
var src source
err := w.db.QueryRow(ctx, `
select coalesce(staging_key, ''), content_type, sha256, size_bytes from attachments where id = $1::uuid`,
job.attachmentID).Scan(&src.stagingKey, &src.contentType, &src.sha256, &src.size)
if err != nil {
return err
}
data, err := w.fetch(ctx, src)
if err != nil {
return err
}
variants, err := renderVariants(ctx, data, w.maxPixels)
if err != nil {
return err
}
original := finalKey(job.attachmentID)
if err := w.store.Put(ctx, original, bytes.NewReader(data), int64(len(data)), src.contentType); err != nil {
return err
}
for _, v := range variants {
if err := w.store.Put(ctx, variantKey(job.attachmentID, v.spec), bytes.NewReader(v.data), int64(len(v.data)), v.spec.contentType); err != nil {
return err
}
}
w.inject("after_store")
tx, err := w.db.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx)
tag, err := tx.Exec(ctx, `
update jobs set status = 'ready', lease_owner = null, lease_until = null, last_error = null, updated_at = now()
where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`, job.id, job.owner, job.attempt)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
w.log.Warn("lease lost, result discarded", "job_id", job.id, "attempt", job.attempt)
return nil
}
for _, v := range variants {
_, err := tx.Exec(ctx, `
insert into attachment_variants(attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256)
values($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9)
on conflict (attachment_id, pipeline_version, name) do update
set object_key = excluded.object_key, content_type = excluded.content_type, size_bytes = excluded.size_bytes,
width = excluded.width, height = excluded.height, sha256 = excluded.sha256`,
job.attachmentID, pipelineVersion, v.spec.name, variantKey(job.attachmentID, v.spec), v.spec.contentType,
len(v.data), v.width, v.height, v.sha256)
if err != nil {
return err
}
}
if _, err := tx.Exec(ctx, `
update attachments set status = 'ready', error_code = null, object_key = $2, updated_at = now()
where id = $1::uuid`, job.attachmentID, original); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
w.log.Info("job ready", "job_id", job.id, "attachment_id", job.attachmentID, "attempt", job.attempt)
w.inject("before_ack")
if err := w.store.Remove(ctx, src.stagingKey); err != nil {
w.log.Warn("staging cleanup deferred to janitor", "attachment_id", job.attachmentID, "err", err.Error())
}
return nil
}
func (w *Worker) settle(ctx context.Context, job *claimedJob, permanent bool, code, reason string) error {
tx, err := w.db.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx)
var maxAttempts int
if err := tx.QueryRow(ctx, `select max_attempts from jobs where id = $1::uuid`, job.id).Scan(&maxAttempts); err != nil {
return err
}
fence := ` where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`
if !permanent && job.attempt < maxAttempts {
delay := w.backoff(job.attempt)
tag, err := tx.Exec(ctx, `
update jobs set status = 'queued', attempt = attempt + 1, next_attempt_at = now() + make_interval(secs => $4),
lease_owner = null, lease_until = null, last_error = $5, updated_at = now()`+fence,
job.id, job.owner, job.attempt, delay.Seconds(), reason)
if err != nil || tag.RowsAffected() == 0 {
return err
}
if _, err := tx.Exec(ctx, `
insert into outbox(job_id, attempt, available_at, created_at)
values($1::uuid, $2, now() + make_interval(secs => $3), now())`, job.id, job.attempt+1, delay.Seconds()); err != nil {
return err
}
if _, err := tx.Exec(ctx, `update attachments set status = 'queued', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil {
return err
}
w.log.Warn("job retry scheduled", "job_id", job.id, "attempt", job.attempt, "next_attempt", job.attempt+1,
"delay_ms", delay.Milliseconds(), "reason", reason)
return tx.Commit(ctx)
}
if !permanent {
code = "retry_exhausted"
}
tag, err := tx.Exec(ctx, `
update jobs set status = 'failed', lease_owner = null, lease_until = null, last_error = $4, updated_at = now()`+fence,
job.id, job.owner, job.attempt, reason)
if err != nil || tag.RowsAffected() == 0 {
return err
}
if _, err := tx.Exec(ctx, `
update attachments set status = 'failed', error_code = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, code); err != nil {
return err
}
w.log.Error("job failed", "job_id", job.id, "attempt", job.attempt, "error_code", code, "reason", reason)
return tx.Commit(ctx)
}
func (w *Worker) handle(ctx context.Context, msg redis.XMessage) bool {
var m jobMessage
raw, _ := msg.Values["job"].(string)
if err := json.Unmarshal([]byte(raw), &m); err != nil || m.SchemaVersion != 1 || m.Type != jobType || !isUUID(m.JobID) {
w.log.Warn("invalid job message dropped", "stream_id", msg.ID)
return true
}
job, err := w.claim(ctx, m.JobID)
if err != nil {
w.log.Warn("claim failed", "job_id", m.JobID, "err", err.Error())
return false
}
if job == nil {
var status string
var attempt int
_ = w.db.QueryRow(ctx, `select status, attempt from jobs where id = $1::uuid`, m.JobID).Scan(&status, &attempt)
w.log.Info("job delivery skipped", "job_id", m.JobID, "message_attempt", m.Attempt, "status", status, "attempt", attempt)
return true
}
err = w.execute(ctx, job)
if err == nil {
return true
}
permanent, code := false, "processing_failed"
switch {
case errors.Is(err, errInvalidImage):
permanent, code = true, "invalid_image"
case errors.Is(err, errSource):
permanent = true
}
if err := w.settle(ctx, job, permanent, code, err.Error()); err != nil {
w.log.Warn("settle failed", "job_id", job.id, "err", err.Error())
return false
}
return true
}
func (w *Worker) read(ctx context.Context, consumer, id string, block time.Duration) ([]redis.XMessage, error) {
res, err := w.broker.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: jobGroup, Consumer: consumer, Streams: []string{jobStream, id}, Count: 1, Block: block,
}).Result()
if len(res) == 0 {
return nil, err
}
return res[0].Messages, err
}
func (w *Worker) consume(ctx context.Context, consumer string) {
pending := true
nextClaim := time.Now().Add(w.lease)
for ctx.Err() == nil {
var msgs []redis.XMessage
var err error
switch {
case pending:
msgs, err = w.read(ctx, consumer, "0", -1)
pending = len(msgs) > 0
case time.Now().After(nextClaim):
msgs, _, err = w.broker.rdb.XAutoClaim(ctx, &redis.XAutoClaimArgs{
Stream: jobStream, Group: jobGroup, Consumer: consumer, MinIdle: w.lease, Start: "0-0", Count: 1,
}).Result()
nextClaim = time.Now().Add(15 * time.Second)
default:
msgs, err = w.read(ctx, consumer, ">", 5*time.Second)
}
if err != nil && !errors.Is(err, redis.Nil) {
if ctx.Err() != nil {
return
}
if strings.Contains(err.Error(), "NOGROUP") {
_ = w.broker.ensureGroup(ctx)
}
w.log.Warn("broker read failed", "consumer", consumer, "err", err.Error())
time.Sleep(2 * time.Second)
continue
}
for _, msg := range msgs {
if !w.handle(context.WithoutCancel(ctx), msg) {
pending = false
continue
}
if err := w.broker.rdb.XAck(context.WithoutCancel(ctx), jobStream, jobGroup, msg.ID).Err(); err != nil {
w.log.Warn("ack failed", "stream_id", msg.ID, "err", err.Error())
}
}
}
}
func (w *Worker) reap(ctx context.Context) error {
rows, err := w.db.Query(ctx, `
select id::text, attachment_id::text, attempt, coalesce(lease_owner, '')
from jobs where status = 'processing' and lease_until < now() limit 50`)
if err != nil {
return err
}
jobs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*claimedJob, error) {
j := &claimedJob{}
return j, row.Scan(&j.id, &j.attachmentID, &j.attempt, &j.owner)
})
if err != nil {
return err
}
for _, j := range jobs {
w.log.Warn("lease expired", "job_id", j.id, "attempt", j.attempt)
if err := w.settle(ctx, j, false, "", "lease expired"); err != nil {
return err
}
}
return nil
}
func runWorker(db *pgxpool.Pool, store *Store, broker *Broker, log *slog.Logger, w *Worker, concurrency int) {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
for {
err := broker.ensureGroup(ctx)
if err == nil {
break
}
log.Warn("broker not ready", "err", err.Error())
select {
case <-ctx.Done():
return
case <-time.After(2 * time.Second):
}
}
host, _ := os.Hostname()
var wg sync.WaitGroup
wg.Go(func() {
for ctx.Err() == nil {
if err := w.reap(ctx); err != nil && ctx.Err() == nil {
log.Warn("lease reaper failed", "err", err.Error())
}
select {
case <-ctx.Done():
case <-time.After(5 * time.Second):
}
}
})
for i := range concurrency {
consumer := fmt.Sprintf("%s-%d", host, i)
wg.Go(func() { w.consume(ctx, consumer) })
}
log.Info("worker started", "owner", w.owner, "concurrency", concurrency, "lease_s", w.lease.Seconds())
<-ctx.Done()
log.Info("worker stopping")
wg.Wait()
}
func replayJob(ctx context.Context, db *pgxpool.Pool, attachmentID string, maxAttempts int) (string, error) {
tx, err := db.Begin(ctx)
if err != nil {
return "", err
}
defer tx.Rollback(ctx)
var jobID, status string
var attempt int
var lastErr *string
err = tx.QueryRow(ctx, `select id::text, status, attempt, last_error from jobs where attachment_id = $1::uuid for update`,
attachmentID).Scan(&jobID, &status, &attempt, &lastErr)
if errors.Is(err, pgx.ErrNoRows) {
return "not_found", nil
}
if err != nil {
return "", err
}
if status != "failed" {
return "skipped_" + status, nil
}
if _, err := tx.Exec(ctx, `insert into job_replays(job_id, previous_error, replayed_at) values($1::uuid, $2, now())`, jobID, lastErr); err != nil {
return "", err
}
if _, err := tx.Exec(ctx, `
update jobs set status = 'queued', attempt = attempt + 1, max_attempts = attempt + $2, next_attempt_at = now(),
last_error = null, updated_at = now()
where id = $1::uuid`, jobID, maxAttempts); err != nil {
return "", err
}
if _, err := tx.Exec(ctx, `insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now(), now())`, jobID, attempt+1); err != nil {
return "", err
}
if _, err := tx.Exec(ctx, `update attachments set status = 'queued', error_code = null, updated_at = now() where id = $1::uuid`, attachmentID); err != nil {
return "", err
}
return "queued", tx.Commit(ctx)
}
func listFailed(ctx context.Context, db *pgxpool.Pool) ([]map[string]any, error) {
rows, err := db.Query(ctx, `
select j.attachment_id::text, j.id::text, j.attempt, coalesce(a.error_code, ''), coalesce(j.last_error, ''), j.updated_at
from jobs j join attachments a on a.id = j.attachment_id
where j.status = 'failed' order by j.updated_at`)
if err != nil {
return nil, err
}
return pgx.CollectRows(rows, func(row pgx.CollectableRow) (map[string]any, error) {
var att, job, code, reason string
var attempt int
var updated time.Time
err := row.Scan(&att, &job, &attempt, &code, &reason, &updated)
return map[string]any{"attachment_id": att, "job_id": job, "attempt": attempt, "error_code": code,
"last_error": reason, "failed_at": formatTime(updated)}, err
})
}