317 lines
10 KiB
Go
317 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"github.com/redis/go-redis/v9"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
otellog "go.opentelemetry.io/otel/log"
|
|
"go.opentelemetry.io/otel/metric"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
)
|
|
|
|
const imageStream = "attachment-jobs"
|
|
const imageGroup = "image-workers"
|
|
|
|
var ackImage = redis.NewScript(`
|
|
if redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) == 1 then
|
|
redis.call('XDEL', KEYS[1], ARGV[2])
|
|
end
|
|
return 1
|
|
`)
|
|
|
|
func imageCommand(db *pgxpool.Pool, dbTrace *databaseTracer, args []string) error {
|
|
a := &App{db: db}
|
|
retention := positive("STAGING_RETENTION_SECONDS", 86400)
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
if args[0] == "jobs" {
|
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
if len(args) == 2 && args[1] == "failed" {
|
|
return a.listFailedImages(ctx)
|
|
}
|
|
if len(args) != 3 {
|
|
return errors.New("usage: jobs failed | jobs replay <job-id> | jobs upgrade <attachment-id>")
|
|
}
|
|
id, err := normalizeID(args[2])
|
|
if err != nil {
|
|
return errors.New("invalid job or attachment ID")
|
|
}
|
|
switch args[1] {
|
|
case "replay":
|
|
return a.replayImage(ctx, id, retention)
|
|
case "upgrade":
|
|
return a.upgradeImage(ctx, id)
|
|
default:
|
|
return errors.New("unknown jobs command")
|
|
}
|
|
}
|
|
|
|
if len(args) != 1 || (args[0] != "dispatcher" && args[0] != "worker") {
|
|
return errors.New("unknown command")
|
|
}
|
|
attempts := positive("JOB_MAX_ATTEMPTS", 5)
|
|
timeoutSeconds := positive("JOB_TIMEOUT_SECONDS", 30)
|
|
if attempts > 20 || timeoutSeconds > 120 || positive("WORKER_CONCURRENCY", 1) != 1 {
|
|
return errors.New("invalid image job limits")
|
|
}
|
|
timeout := time.Duration(timeoutSeconds) * time.Second
|
|
s3, bucket, err := s3Client()
|
|
if err != nil {
|
|
return errors.New("invalid image storage configuration")
|
|
}
|
|
a.s3, a.bucket = s3, bucket
|
|
brokerURL, err := secretFile("BROKER_URL_FILE")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
opts, err := redis.ParseURL(brokerURL)
|
|
if err != nil {
|
|
return errors.New("invalid broker configuration")
|
|
}
|
|
opts.MaxRetries = -1
|
|
opts.PoolSize = 2
|
|
opts.DialTimeout = time.Second
|
|
opts.ReadTimeout = 2 * time.Second
|
|
opts.WriteTimeout = time.Second
|
|
opts.ContextTimeoutEnabled = true
|
|
broker := redis.NewClient(opts)
|
|
defer broker.Close()
|
|
instruments, shutdown, err := startTelemetry(args[0], db, dbTrace)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer shutdown()
|
|
a.telemetry = instruments
|
|
|
|
if args[0] == "dispatcher" {
|
|
a.dispatchImages(ctx, broker, attempts, retention)
|
|
} else {
|
|
a.consumeImages(ctx, broker, attempts, timeout)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func injectImageFault(phase, attachment string) {
|
|
if os.Getenv("LAB5_FAULT_ATTACHMENT") == attachment && os.Getenv("LAB5_FAULT") == phase {
|
|
slog.Info("test crash", "phase", phase, "attachment_id", attachment)
|
|
os.Exit(86)
|
|
}
|
|
}
|
|
|
|
func (a *App) publishImage(ctx context.Context, broker *redis.Client) (published bool, result error) {
|
|
tx, err := a.db.Begin(ctx)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
defer rollbackAttachment(tx)
|
|
j := imageJob{SchemaVersion: 1, Type: "attachment.process.v1"}
|
|
var created time.Time
|
|
var parent, state string
|
|
err = tx.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt,created_at,traceparent,tracestate FROM image_jobs
|
|
WHERE state='queued' AND NOT published AND next_at<=clock_timestamp()
|
|
ORDER BY next_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &created, &parent, &state)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
j.CreatedAt = stamp(created)
|
|
payload, err := json.Marshal(j)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
ctx, span := a.telemetry.startJob(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state}, j, true)
|
|
defer func() { endSpan(span, result) }()
|
|
carrier := propagation.MapCarrier{}
|
|
traceContext.Inject(ctx, carrier)
|
|
values := map[string]any{"job": string(payload)}
|
|
for key, value := range carrier {
|
|
values[key] = value
|
|
}
|
|
injectImageFault("before_publish", j.AttachmentID)
|
|
// XADD replies after the broker's AOF durability boundary. A lost reply
|
|
// or crash before the SQL commit causes another delivery of the same job.
|
|
err = broker.XAdd(ctx, &redis.XAddArgs{Stream: imageStream, Values: values}).Err()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
injectImageFault("after_publish", j.AttachmentID)
|
|
if _, err = tx.Exec(ctx, "UPDATE image_jobs SET published=true WHERE id=$1", j.JobID); err != nil {
|
|
return false, err
|
|
}
|
|
err = tx.Commit(ctx)
|
|
if err == nil {
|
|
a.telemetry.event(ctx, "image.published", otellog.SeverityInfo,
|
|
otellog.String("job_id", j.JobID), otellog.Int("attempt", j.Attempt))
|
|
}
|
|
return true, err
|
|
}
|
|
|
|
func (a *App) dispatchImages(ctx context.Context, broker *redis.Client, attempts, retention int) {
|
|
cleanup := time.Now()
|
|
for ctx.Err() == nil {
|
|
step, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
recoverErr := a.recoverImage(step, attempts)
|
|
published, publishErr := a.publishImage(step, broker)
|
|
cancel()
|
|
if recoverErr != nil || publishErr != nil {
|
|
slog.Warn("image outbox unavailable", "recover_failed", recoverErr != nil, "publish_failed", publishErr != nil)
|
|
}
|
|
if time.Now().After(cleanup) {
|
|
step, cancel = context.WithTimeout(ctx, 5*time.Second)
|
|
if err := a.cleanStaging(step, retention); err != nil && ctx.Err() == nil {
|
|
slog.Warn("staging cleanup incomplete")
|
|
}
|
|
cancel()
|
|
cleanup = time.Now().Add(15 * time.Second)
|
|
}
|
|
if !published || recoverErr != nil {
|
|
waitImage(ctx, 300*time.Millisecond)
|
|
}
|
|
}
|
|
}
|
|
|
|
func waitImage(ctx context.Context, delay time.Duration) {
|
|
timer := time.NewTimer(delay)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
|
|
func (a *App) consumeImages(ctx context.Context, broker *redis.Client, attempts int, timeout time.Duration) {
|
|
cursor := "0-0"
|
|
groupReady := false
|
|
for ctx.Err() == nil {
|
|
if !groupReady {
|
|
err := broker.XGroupCreateMkStream(ctx, imageStream, imageGroup, "0").Err()
|
|
if err != nil && !strings.HasPrefix(err.Error(), "BUSYGROUP") {
|
|
waitImage(ctx, time.Second)
|
|
continue
|
|
}
|
|
groupReady = true
|
|
}
|
|
messages, next, err := broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
|
Stream: imageStream, Group: imageGroup, Consumer: "worker", MinIdle: timeout + 10*time.Second, Start: cursor, Count: 1,
|
|
}).Result()
|
|
if err == nil {
|
|
cursor = next
|
|
if len(messages) == 0 {
|
|
var streams []redis.XStream
|
|
streams, err = broker.XReadGroup(ctx, &redis.XReadGroupArgs{
|
|
Group: imageGroup, Consumer: "worker", Streams: []string{imageStream, ">"}, Count: 1, Block: time.Second,
|
|
}).Result()
|
|
if err == nil && len(streams) > 0 {
|
|
messages = streams[0].Messages
|
|
}
|
|
}
|
|
}
|
|
if err != nil && !errors.Is(err, redis.Nil) {
|
|
groupReady = false
|
|
waitImage(ctx, time.Second)
|
|
continue
|
|
}
|
|
for _, message := range messages {
|
|
if err = a.handleImage(ctx, message, attempts, timeout); err != nil {
|
|
slog.Warn("image delivery left pending", "delivery_id", message.ID)
|
|
continue
|
|
}
|
|
if err = ackImage.Run(ctx, broker, []string{imageStream}, imageGroup, message.ID).Err(); err != nil {
|
|
slog.Warn("image acknowledgement unavailable", "delivery_id", message.ID)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *App) handleImage(ctx context.Context, message redis.XMessage, attempts int, timeout time.Duration) (result error) {
|
|
var j imageJob
|
|
payload, ok := message.Values["job"].(string)
|
|
if !ok || len(payload) > 2048 || json.Unmarshal([]byte(payload), &j) != nil ||
|
|
j.SchemaVersion != 1 || j.Type != "attachment.process.v1" || j.Attempt < 1 || (j.Version != 1 && j.Version != 2) {
|
|
slog.Warn("invalid image delivery", "delivery_id", message.ID)
|
|
return nil
|
|
}
|
|
for _, id := range []string{j.JobID, j.AttachmentID} {
|
|
if normalized, err := normalizeID(id); err != nil || normalized != id {
|
|
slog.Warn("invalid image delivery ID", "delivery_id", message.ID)
|
|
return nil
|
|
}
|
|
}
|
|
parent, _ := message.Values["traceparent"].(string)
|
|
state, _ := message.Values["tracestate"].(string)
|
|
ctx, span := a.telemetry.startJob(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state}, j, false)
|
|
defer func() { endSpan(span, result) }()
|
|
claimCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
src, claimed, err := a.claimImage(claimCtx, j, timeout+5*time.Second)
|
|
cancel()
|
|
if err != nil || !claimed {
|
|
if err != nil {
|
|
a.telemetry.event(ctx, "image.claim_failed", otellog.SeverityError, otellog.String("job_id", j.JobID))
|
|
}
|
|
return err // Stale/duplicate attempts are no-ops. A processing row has a recoverable lease.
|
|
}
|
|
outcome := "error"
|
|
started := time.Now()
|
|
if a.telemetry != nil {
|
|
a.telemetry.workerBusy.Add(1)
|
|
defer a.telemetry.workerBusy.Add(-1)
|
|
}
|
|
defer func() { a.telemetry.finishJob(ctx, j, outcome, started) }()
|
|
injectImageFault("before_effect", j.AttachmentID)
|
|
processCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
var variants []Variant
|
|
switch {
|
|
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "invalid_image":
|
|
err = imageFailure("invalid_image")
|
|
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID &&
|
|
(os.Getenv("LAB5_FAULT") == "temporary" || (os.Getenv("LAB5_FAULT") == "temporary_once" && j.Attempt == 1)):
|
|
err = errors.New("test transient failure")
|
|
default:
|
|
variants, err = a.generateImage(processCtx, j, src)
|
|
}
|
|
cancel()
|
|
finishCtx, cancel := context.WithTimeout(ctx, 4*time.Second)
|
|
defer cancel()
|
|
if err != nil {
|
|
code := "processing_failed"
|
|
var permanent imageFailure
|
|
if errors.As(err, &permanent) {
|
|
code = string(permanent)
|
|
}
|
|
next, saveErr := a.failImage(finishCtx, j, code, permanent != "", false, attempts)
|
|
if saveErr == nil {
|
|
outcome = next
|
|
}
|
|
span.SetAttributes(attribute.String("error.type", code))
|
|
return saveErr
|
|
}
|
|
seconds, committed, err := a.completeImage(finishCtx, j, variants)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !committed {
|
|
outcome = "lease_lost"
|
|
return nil
|
|
}
|
|
outcome = "ready"
|
|
if a.telemetry != nil && j.Version == 1 {
|
|
a.telemetry.readyTime.Record(ctx, seconds, metric.WithAttributes(attribute.String("job_type", j.Type)))
|
|
}
|
|
injectImageFault("after_commit", j.AttachmentID)
|
|
return nil
|
|
}
|