package main import ( "context" "encoding/json" "errors" "log/slog" "os" "os/signal" "strings" "syscall" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" ) const imageStream = "attachment-jobs" const imageGroup = "image-workers" var ackImage = redis.NewScript(` if redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) == 1 then redis.call('XDEL', KEYS[1], ARGV[2]) end return 1 `) func imageCommand(db *pgxpool.Pool, args []string) error { a := &App{db: db} retention := positive("STAGING_RETENTION_SECONDS", 86400) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() if args[0] == "jobs" { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() if len(args) == 2 && args[1] == "failed" { return a.listFailedImages(ctx) } if len(args) != 3 { return errors.New("usage: jobs failed | jobs replay | jobs upgrade ") } id, err := normalizeID(args[2]) if err != nil { return errors.New("invalid job or attachment ID") } switch args[1] { case "replay": return a.replayImage(ctx, id, retention) case "upgrade": return a.upgradeImage(ctx, id) default: return errors.New("unknown jobs command") } } if len(args) != 1 || (args[0] != "dispatcher" && args[0] != "worker") { return errors.New("unknown command") } attempts := positive("JOB_MAX_ATTEMPTS", 5) timeoutSeconds := positive("JOB_TIMEOUT_SECONDS", 30) if attempts > 20 || timeoutSeconds > 120 || positive("WORKER_CONCURRENCY", 1) != 1 { return errors.New("invalid image job limits") } timeout := time.Duration(timeoutSeconds) * time.Second s3, bucket, err := s3Client() if err != nil { return errors.New("invalid image storage configuration") } a.s3, a.bucket = s3, bucket brokerURL, err := secretFile("BROKER_URL_FILE") if err != nil { return err } opts, err := redis.ParseURL(brokerURL) if err != nil { return errors.New("invalid broker configuration") } opts.MaxRetries = -1 opts.PoolSize = 2 opts.DialTimeout = time.Second opts.ReadTimeout = 2 * time.Second opts.WriteTimeout = time.Second opts.ContextTimeoutEnabled = true broker := redis.NewClient(opts) defer broker.Close() if args[0] == "dispatcher" { a.dispatchImages(ctx, broker, attempts, retention) } else { a.consumeImages(ctx, broker, attempts, timeout) } return nil } func injectImageFault(phase, attachment string) { if os.Getenv("LAB5_FAULT_ATTACHMENT") == attachment && os.Getenv("LAB5_FAULT") == phase { slog.Info("test crash", "phase", phase, "attachment_id", attachment) os.Exit(86) } } func (a *App) publishImage(ctx context.Context, broker *redis.Client) (bool, error) { tx, err := a.db.Begin(ctx) if err != nil { return false, err } defer rollbackAttachment(tx) j := imageJob{SchemaVersion: 1, Type: "attachment.process.v1"} var created time.Time err = tx.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt,created_at FROM image_jobs WHERE state='queued' AND NOT published AND next_at<=clock_timestamp() ORDER BY next_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &created) if errors.Is(err, pgx.ErrNoRows) { return false, nil } if err != nil { return false, err } j.CreatedAt = stamp(created) payload, err := json.Marshal(j) if err != nil { return false, err } injectImageFault("before_publish", j.AttachmentID) // XADD replies after the broker's AOF durability boundary. A lost reply // or crash before the SQL commit causes another delivery of the same job. err = broker.XAdd(ctx, &redis.XAddArgs{Stream: imageStream, Values: map[string]any{"job": string(payload)}}).Err() if err != nil { return false, err } injectImageFault("after_publish", j.AttachmentID) if _, err = tx.Exec(ctx, "UPDATE image_jobs SET published=true WHERE id=$1", j.JobID); err != nil { return false, err } return true, tx.Commit(ctx) } func (a *App) dispatchImages(ctx context.Context, broker *redis.Client, attempts, retention int) { cleanup := time.Now() for ctx.Err() == nil { step, cancel := context.WithTimeout(ctx, 5*time.Second) recoverErr := a.recoverImage(step, attempts) published, publishErr := a.publishImage(step, broker) cancel() if recoverErr != nil || publishErr != nil { slog.Warn("image outbox unavailable", "recover_failed", recoverErr != nil, "publish_failed", publishErr != nil) } if time.Now().After(cleanup) { step, cancel = context.WithTimeout(ctx, 5*time.Second) if err := a.cleanStaging(step, retention); err != nil && ctx.Err() == nil { slog.Warn("staging cleanup incomplete") } cancel() cleanup = time.Now().Add(15 * time.Second) } if !published || recoverErr != nil { waitImage(ctx, 300*time.Millisecond) } } } func waitImage(ctx context.Context, delay time.Duration) { timer := time.NewTimer(delay) defer timer.Stop() select { case <-ctx.Done(): case <-timer.C: } } func (a *App) consumeImages(ctx context.Context, broker *redis.Client, attempts int, timeout time.Duration) { cursor := "0-0" groupReady := false for ctx.Err() == nil { if !groupReady { err := broker.XGroupCreateMkStream(ctx, imageStream, imageGroup, "0").Err() if err != nil && !strings.HasPrefix(err.Error(), "BUSYGROUP") { waitImage(ctx, time.Second) continue } groupReady = true } messages, next, err := broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{ Stream: imageStream, Group: imageGroup, Consumer: "worker", MinIdle: timeout + 10*time.Second, Start: cursor, Count: 1, }).Result() if err == nil { cursor = next if len(messages) == 0 { var streams []redis.XStream streams, err = broker.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: imageGroup, Consumer: "worker", Streams: []string{imageStream, ">"}, Count: 1, Block: time.Second, }).Result() if err == nil && len(streams) > 0 { messages = streams[0].Messages } } } if err != nil && !errors.Is(err, redis.Nil) { groupReady = false waitImage(ctx, time.Second) continue } for _, message := range messages { if err = a.handleImage(ctx, message, attempts, timeout); err != nil { slog.Warn("image delivery left pending", "delivery_id", message.ID) continue } if err = ackImage.Run(ctx, broker, []string{imageStream}, imageGroup, message.ID).Err(); err != nil { slog.Warn("image acknowledgement unavailable", "delivery_id", message.ID) } } } } func (a *App) handleImage(ctx context.Context, message redis.XMessage, attempts int, timeout time.Duration) error { var j imageJob payload, ok := message.Values["job"].(string) if !ok || len(payload) > 2048 || json.Unmarshal([]byte(payload), &j) != nil || j.SchemaVersion != 1 || j.Type != "attachment.process.v1" || j.Attempt < 1 || (j.Version != 1 && j.Version != 2) { slog.Warn("invalid image delivery", "delivery_id", message.ID) return nil } for _, id := range []string{j.JobID, j.AttachmentID} { if normalized, err := normalizeID(id); err != nil || normalized != id { slog.Warn("invalid image delivery ID", "delivery_id", message.ID) return nil } } claimCtx, cancel := context.WithTimeout(ctx, 5*time.Second) src, claimed, err := a.claimImage(claimCtx, j, timeout+5*time.Second) cancel() if err != nil || !claimed { return err // Stale/duplicate attempts are no-ops. A processing row has a recoverable lease. } injectImageFault("before_effect", j.AttachmentID) processCtx, cancel := context.WithTimeout(ctx, timeout) var variants []Variant switch { case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "invalid_image": err = imageFailure("invalid_image") case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "temporary": err = errors.New("test transient failure") default: variants, err = a.generateImage(processCtx, j, src) } cancel() finishCtx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() if err != nil { code := "processing_failed" var permanent imageFailure if errors.As(err, &permanent) { code = string(permanent) } return a.failImage(finishCtx, j, code, permanent != "", false, attempts) } if err = a.completeImage(finishCtx, j, variants); err != nil { return err } injectImageFault("after_commit", j.AttachmentID) return nil }