diff --git a/internal/worker/image.go b/internal/worker/image.go index 2f2bccae..e7fac0bd 100644 --- a/internal/worker/image.go +++ b/internal/worker/image.go @@ -66,21 +66,21 @@ func thumbnail(r io.ReadSeeker, maxPixels int) ([]byte, int, int, error) { return out.Bytes(), width, height, nil } -func (w *Worker) process(ctx context.Context, row db.Attachment) (db.CreateAttachmentVariantParams, error) { +func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAttachmentVariantParams, error) { if row.StagingKey == nil { - return db.CreateAttachmentVariantParams{}, errInvalidSource + return nil, errInvalidSource } source, err := w.s3.Get(ctx, *row.StagingKey) if err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } defer source.Body.Close() tmp, err := os.CreateTemp("", "attachment-job-*") if err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } defer func() { @@ -91,54 +91,54 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) (db.CreateAttac hash := sha256.New() size, err := io.Copy(io.MultiWriter(tmp, hash), io.LimitReader(source.Body, int64(w.maxBytes)+1)) if err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } if size != int64(row.SizeBytes) || hex.EncodeToString(hash.Sum(nil)) != row.Sha256 { - return db.CreateAttachmentVariantParams{}, errInvalidSource + return nil, errInvalidSource } if _, err := tmp.Seek(0, io.SeekStart); err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } select { case w.decodeSlot <- struct{}{}: case <-ctx.Done(): - return db.CreateAttachmentVariantParams{}, ctx.Err() + return nil, ctx.Err() } if err := ctx.Err(); err != nil { <-w.decodeSlot - return db.CreateAttachmentVariantParams{}, err + return nil, err } data, width, height, err := thumbnail(tmp, w.maxPixels) <-w.decodeSlot if err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } if err := ctx.Err(); err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } if _, err := tmp.Seek(0, io.SeekStart); err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } if err := w.s3.Put(ctx, row.S3Key, tmp, size, string(row.ContentType)); err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } key := "variants/" + row.ID.String() + "/v" + strconv.Itoa(row.PipelineVersion) + "/thumbnail" if err := w.s3.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "image/jpeg"); err != nil { - return db.CreateAttachmentVariantParams{}, err + return nil, err } sum := sha256.Sum256(data) - return db.CreateAttachmentVariantParams{ + return []db.CreateAttachmentVariantParams{{ AttachmentID: row.ID, PipelineVersion: row.PipelineVersion, Variant: db.AttachmentVariantNameThumbnail, @@ -148,5 +148,5 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) (db.CreateAttac Width: width, Height: height, Sha256: hex.EncodeToString(sum[:]), - }, nil + }}, nil } diff --git a/internal/worker/job.go b/internal/worker/job.go new file mode 100644 index 00000000..66b7775b --- /dev/null +++ b/internal/worker/job.go @@ -0,0 +1,218 @@ +package worker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "time" + + "github.com/jackc/pgx/v5" + + "mtusi2026-messenger/internal/db" + "mtusi2026-messenger/internal/model" +) + +var ( + errLeaseExpired = errors.New("attachment lease expired") + errMissingThumbnail = errors.New("ready attachment has no thumbnail") + errAttemptProcessing = errors.New("attachment attempt is still processing") + errAttemptUnavailable = errors.New("attachment attempt is not yet available") + errEffectChecksumMismatch = errors.New("attachment effect checksum mismatch") +) + +func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error { + jobCtx, cancel := context.WithTimeout(ctx, w.cfg.Timeout) + defer cancel() + + row, err := w.q.ClaimAttachmentJob(jobCtx, db.ClaimAttachmentJobParams{ + ID: job.AttachmentID, + JobID: job.JobID, + PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, + LeaseUntil: time.Now().Add(w.cfg.Timeout), + }) + if errors.Is(err, pgx.ErrNoRows) { + return w.duplicate(jobCtx, job) + } + + if err != nil { + return err + } + + w.log.Info("attachment processing", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt) + variants, processErr := w.process(jobCtx, row) + if processErr != nil { + resultCtx, resultCancel := context.WithTimeout(ctx, 5*time.Second) + defer resultCancel() + + return w.finishFailure(resultCtx, job, processErr) + } + + tx, err := w.pool.Begin(jobCtx) + if err != nil { + return err + } + + defer tx.Rollback(context.Background()) + + q := w.q.WithTx(tx) + _, err = q.LockAttachmentJob(jobCtx, db.LockAttachmentJobParams{ + ID: job.AttachmentID, + JobID: job.JobID, + PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, + }) + if err != nil { + return err + } + + for _, variant := range variants { + if err := q.CreateAttachmentVariant(jobCtx, variant); err != nil { + return err + } + } + + n, err := q.CompleteAttachmentJob(jobCtx, db.CompleteAttachmentJobParams{ + ID: row.ID, + Attempt: job.Attempt, + }) + if err != nil { + return err + } + + if n != 1 { + return errLeaseExpired + } + + if err := tx.Commit(jobCtx); err != nil { + return err + } + + if row.StagingKey != nil { + err := w.deleteStagingObject(ctx, *row.StagingKey) + if err != nil && ctx.Err() == nil { + w.log.Warn("delete ready attachment staging", "attachment_id", row.ID, "error", err) + } + } + + w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt) + return nil +} + +func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error { + row, err := w.q.GetAttachmentByID(ctx, job.AttachmentID) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + + if err != nil { + return err + } + + if row.JobID == nil || *row.JobID != job.JobID || row.PipelineVersion != job.PipelineVersion { + return nil + } + + if row.Status == db.AttachmentStatusFailed { + return nil + } + + if row.Status == db.AttachmentStatusReady { + variants, err := w.q.ListAttachmentVariants(ctx, db.ListAttachmentVariantsParams{ + AttachmentID: row.ID, + PipelineVersion: row.PipelineVersion, + }) + if err != nil { + return err + } + + if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil { + return err + } + + hasThumbnail := false + for _, variant := range variants { + if err := w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil { + return err + } + + if variant.Variant == db.AttachmentVariantNameThumbnail { + hasThumbnail = true + } + } + + if !hasThumbnail { + return errMissingThumbnail + } + + return nil + } + + if row.Status == db.AttachmentStatusProcessing && row.Attempts == job.Attempt { + return errAttemptProcessing + } + + if row.Attempts >= job.Attempt { + return nil + } + + return errAttemptUnavailable +} + +func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error { + object, err := w.s3.Get(ctx, key) + if err != nil { + return err + } + + defer object.Body.Close() + + hash := sha256.New() + n, err := io.Copy(hash, io.LimitReader(object.Body, int64(size)+1)) + if err != nil { + return err + } + + if n != int64(size) || hex.EncodeToString(hash.Sum(nil)) != checksum { + return errEffectChecksumMismatch + } + + return nil +} + +func (w *Worker) deleteStagingObject(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + tx, err := w.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + + q := w.q.WithTx(tx) + row, err := q.LockAttachmentByStagingKey(ctx, key) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + + if err != nil { + return err + } + + if row.Status != db.AttachmentStatusReady { + return nil + } + + if err := w.s3.Delete(ctx, key); err != nil { + return err + } + + if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil { + return err + } + + return tx.Commit(ctx) +} diff --git a/internal/worker/retry.go b/internal/worker/retry.go new file mode 100644 index 00000000..52359624 --- /dev/null +++ b/internal/worker/retry.go @@ -0,0 +1,150 @@ +package worker + +import ( + "context" + "errors" + "math/rand/v2" + "time" + + "mtusi2026-messenger/internal/db" + "mtusi2026-messenger/internal/model" + s3store "mtusi2026-messenger/internal/s3" +) + +func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cause error) error { + tx, err := w.pool.Begin(ctx) + if err != nil { + return err + } + + defer tx.Rollback(context.Background()) + + q := w.q.WithTx(tx) + row, err := q.LockAttachmentJob(ctx, db.LockAttachmentJobParams{ + ID: job.AttachmentID, + JobID: job.JobID, + PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, + }) + if err != nil { + return err + } + + code := "processing_failed" + permanent := errors.Is(cause, errInvalidSource) || errors.Is(cause, s3store.ErrObjectNotFound) + if permanent { + code = "invalid_source" + } else if errors.Is(cause, context.DeadlineExceeded) { + code = "job_timeout" + } + + var n int64 + if permanent || row.Attempts >= w.cfg.MaxAttempts { + if !permanent { + code = "retry_exhausted" + } + + n, err = q.FailAttachmentJob(ctx, db.FailAttachmentJobParams{ + ID: row.ID, + Attempt: job.Attempt, + ErrorCode: code, + }) + } else { + n, err = q.RetryAttachmentJob(ctx, db.RetryAttachmentJobParams{ + ID: row.ID, + Attempt: job.Attempt, + ErrorCode: code, + }) + if err == nil && n == 1 { + _, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{ + AttachmentID: row.ID, + Attempt: row.Attempts + 1, + AvailableAt: time.Now().Add(retryDelay(row.Attempts)), + }) + } + } + + if err != nil { + return err + } + + if n != 1 { + return errLeaseExpired + } + + if err := tx.Commit(ctx); err != nil { + return err + } + + w.log.Warn( + "attachment processing failed", + "job_id", + job.JobID, + "attempt", + job.Attempt, + "error_code", + code, + ) + return nil +} + +func (w *Worker) recoverLoop(ctx context.Context) { + for ctx.Err() == nil { + recovered, err := w.recoverOne(ctx) + if err != nil && ctx.Err() == nil { + w.log.Warn("recover attachment lease", "error", err) + } + + if (err != nil || !recovered) && !wait(ctx, time.Second) { + return + } + } +} + +func (w *Worker) recoverOne(ctx context.Context) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + tx, err := w.pool.Begin(ctx) + if err != nil { + return false, err + } + + defer tx.Rollback(context.Background()) + + q := w.q.WithTx(tx) + rows, err := q.LockExpiredAttachmentJobs(ctx, 1) + if err != nil || len(rows) == 0 { + return false, err + } + + row := rows[0] + if row.Attempts >= w.cfg.MaxAttempts { + _, err = q.FailExpiredAttachmentJob(ctx, row.ID) + } else { + _, err = q.RecoverExpiredAttachmentJob(ctx, row.ID) + if err == nil { + _, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{ + AttachmentID: row.ID, + Attempt: row.Attempts + 1, + AvailableAt: time.Now().Add(retryDelay(row.Attempts)), + }) + } + } + + if err != nil { + return false, err + } + + if err := tx.Commit(ctx); err != nil { + return false, err + } + + w.log.Info("attachment lease recovered", "attachment_id", row.ID, "attempt", row.Attempts) + return true, nil +} + +func retryDelay(attempt int) time.Duration { + ceiling := time.Second * time.Duration(1<= job.Attempt { - return nil - } - - return errAttemptUnavailable -} - -func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error { - object, err := w.s3.Get(ctx, key) - if err != nil { - return err - } - - defer object.Body.Close() - - hash := sha256.New() - n, err := io.Copy(hash, io.LimitReader(object.Body, int64(size)+1)) - if err != nil { - return err - } - - if n != int64(size) || hex.EncodeToString(hash.Sum(nil)) != checksum { - return errEffectChecksumMismatch - } - - return nil -} - -func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cause error) error { - tx, err := w.pool.Begin(ctx) - if err != nil { - return err - } - - defer tx.Rollback(context.Background()) - - q := w.q.WithTx(tx) - row, err := q.LockAttachmentJob(ctx, db.LockAttachmentJobParams{ - ID: job.AttachmentID, - JobID: job.JobID, - PipelineVersion: job.PipelineVersion, - Attempt: job.Attempt, - }) - if err != nil { - return err - } - - code := "processing_failed" - permanent := errors.Is(cause, errInvalidSource) || errors.Is(cause, s3store.ErrObjectNotFound) - if permanent { - code = "invalid_source" - } else if errors.Is(cause, context.DeadlineExceeded) { - code = "job_timeout" - } - - var n int64 - if permanent || row.Attempts >= w.cfg.MaxAttempts { - if !permanent { - code = "retry_exhausted" - } - - n, err = q.FailAttachmentJob(ctx, db.FailAttachmentJobParams{ - ID: row.ID, - Attempt: job.Attempt, - ErrorCode: code, - }) - } else { - n, err = q.RetryAttachmentJob(ctx, db.RetryAttachmentJobParams{ - ID: row.ID, - Attempt: job.Attempt, - ErrorCode: code, - }) - if err == nil && n == 1 { - _, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{ - AttachmentID: row.ID, - Attempt: row.Attempts + 1, - AvailableAt: time.Now().Add(retryDelay(row.Attempts)), - }) - } - } - - if err != nil { - return err - } - - if n != 1 { - return errLeaseExpired - } - - if err := tx.Commit(ctx); err != nil { - return err - } - - w.log.Warn( - "attachment processing failed", - "job_id", - job.JobID, - "attempt", - job.Attempt, - "error_code", - code, - ) - return nil -} - -func (w *Worker) deleteStagingObject(ctx context.Context, key string) error { - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - tx, err := w.pool.Begin(ctx) - if err != nil { - return err - } - defer tx.Rollback(context.Background()) - - q := w.q.WithTx(tx) - row, err := q.LockAttachmentByStagingKey(ctx, key) - if errors.Is(err, pgx.ErrNoRows) { - return nil - } - - if err != nil { - return err - } - - if row.Status != db.AttachmentStatusReady { - return nil - } - - if err := w.s3.Delete(ctx, key); err != nil { - return err - } - - if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil { - return err - } - - return tx.Commit(ctx) -} - -func (w *Worker) recoverLoop(ctx context.Context) { - for ctx.Err() == nil { - recovered, err := w.recoverOne(ctx) - if err != nil && ctx.Err() == nil { - w.log.Warn("recover attachment lease", "error", err) - } - - if (err != nil || !recovered) && !wait(ctx, time.Second) { - return - } - } -} - -func (w *Worker) recoverOne(ctx context.Context) (bool, error) { - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - tx, err := w.pool.Begin(ctx) - if err != nil { - return false, err - } - - defer tx.Rollback(context.Background()) - - q := w.q.WithTx(tx) - rows, err := q.LockExpiredAttachmentJobs(ctx, 1) - if err != nil || len(rows) == 0 { - return false, err - } - - row := rows[0] - if row.Attempts >= w.cfg.MaxAttempts { - _, err = q.FailExpiredAttachmentJob(ctx, row.ID) - } else { - _, err = q.RecoverExpiredAttachmentJob(ctx, row.ID) - if err == nil { - _, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{ - AttachmentID: row.ID, - Attempt: row.Attempts + 1, - AvailableAt: time.Now().Add(retryDelay(row.Attempts)), - }) - } - } - - if err != nil { - return false, err - } - - if err := tx.Commit(ctx); err != nil { - return false, err - } - - w.log.Info("attachment lease recovered", "attachment_id", row.ID, "attempt", row.Attempts) - return true, nil -} - func wait(ctx context.Context, duration time.Duration) bool { timer := time.NewTimer(duration) defer timer.Stop() @@ -500,8 +169,3 @@ func wait(ctx context.Context, duration time.Duration) bool { return true } } - -func retryDelay(attempt int) time.Duration { - ceiling := time.Second * time.Duration(1<