refactor(worker): split job handling and retries, prepare variant collection
This commit is contained in:
+16
-16
@@ -66,21 +66,21 @@ func thumbnail(r io.ReadSeeker, maxPixels int) ([]byte, int, int, error) {
|
||||
return out.Bytes(), width, height, nil
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, row db.Attachment) (db.CreateAttachmentVariantParams, error) {
|
||||
func (w *Worker) process(ctx context.Context, row db.Attachment) ([]db.CreateAttachmentVariantParams, error) {
|
||||
if row.StagingKey == nil {
|
||||
return db.CreateAttachmentVariantParams{}, errInvalidSource
|
||||
return nil, errInvalidSource
|
||||
}
|
||||
|
||||
source, err := w.s3.Get(ctx, *row.StagingKey)
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer source.Body.Close()
|
||||
|
||||
tmp, err := os.CreateTemp("", "attachment-job-*")
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
@@ -91,54 +91,54 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) (db.CreateAttac
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(io.MultiWriter(tmp, hash), io.LimitReader(source.Body, int64(w.maxBytes)+1))
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if size != int64(row.SizeBytes) ||
|
||||
hex.EncodeToString(hash.Sum(nil)) != row.Sha256 {
|
||||
return db.CreateAttachmentVariantParams{}, errInvalidSource
|
||||
return nil, errInvalidSource
|
||||
}
|
||||
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
select {
|
||||
case w.decodeSlot <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return db.CreateAttachmentVariantParams{}, ctx.Err()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-w.decodeSlot
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, width, height, err := thumbnail(tmp, w.maxPixels)
|
||||
<-w.decodeSlot
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := w.s3.Put(ctx, row.S3Key, tmp, size, string(row.ContentType)); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key := "variants/" + row.ID.String() + "/v" + strconv.Itoa(row.PipelineVersion) + "/thumbnail"
|
||||
if err := w.s3.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "image/jpeg"); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sum := sha256.Sum256(data)
|
||||
return db.CreateAttachmentVariantParams{
|
||||
return []db.CreateAttachmentVariantParams{{
|
||||
AttachmentID: row.ID,
|
||||
PipelineVersion: row.PipelineVersion,
|
||||
Variant: db.AttachmentVariantNameThumbnail,
|
||||
@@ -148,5 +148,5 @@ func (w *Worker) process(ctx context.Context, row db.Attachment) (db.CreateAttac
|
||||
Width: width,
|
||||
Height: height,
|
||||
Sha256: hex.EncodeToString(sum[:]),
|
||||
}, nil
|
||||
}}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
"mtusi2026-messenger/internal/model"
|
||||
)
|
||||
|
||||
var (
|
||||
errLeaseExpired = errors.New("attachment lease expired")
|
||||
errMissingThumbnail = errors.New("ready attachment has no thumbnail")
|
||||
errAttemptProcessing = errors.New("attachment attempt is still processing")
|
||||
errAttemptUnavailable = errors.New("attachment attempt is not yet available")
|
||||
errEffectChecksumMismatch = errors.New("attachment effect checksum mismatch")
|
||||
)
|
||||
|
||||
func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error {
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
row, err := w.q.ClaimAttachmentJob(jobCtx, db.ClaimAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
LeaseUntil: time.Now().Add(w.cfg.Timeout),
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return w.duplicate(jobCtx, job)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Info("attachment processing", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
|
||||
variants, processErr := w.process(jobCtx, row)
|
||||
if processErr != nil {
|
||||
resultCtx, resultCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer resultCancel()
|
||||
|
||||
return w.finishFailure(resultCtx, job, processErr)
|
||||
}
|
||||
|
||||
tx, err := w.pool.Begin(jobCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
_, err = q.LockAttachmentJob(jobCtx, db.LockAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, variant := range variants {
|
||||
if err := q.CreateAttachmentVariant(jobCtx, variant); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
n, err := q.CompleteAttachmentJob(jobCtx, db.CompleteAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return errLeaseExpired
|
||||
}
|
||||
|
||||
if err := tx.Commit(jobCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.StagingKey != nil {
|
||||
err := w.deleteStagingObject(ctx, *row.StagingKey)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
w.log.Warn("delete ready attachment staging", "attachment_id", row.ID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
|
||||
row, err := w.q.GetAttachmentByID(ctx, job.AttachmentID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.JobID == nil || *row.JobID != job.JobID || row.PipelineVersion != job.PipelineVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusFailed {
|
||||
return nil
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusReady {
|
||||
variants, err := w.q.ListAttachmentVariants(ctx, db.ListAttachmentVariantsParams{
|
||||
AttachmentID: row.ID,
|
||||
PipelineVersion: row.PipelineVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasThumbnail := false
|
||||
for _, variant := range variants {
|
||||
if err := w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if variant.Variant == db.AttachmentVariantNameThumbnail {
|
||||
hasThumbnail = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hasThumbnail {
|
||||
return errMissingThumbnail
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusProcessing && row.Attempts == job.Attempt {
|
||||
return errAttemptProcessing
|
||||
}
|
||||
|
||||
if row.Attempts >= job.Attempt {
|
||||
return nil
|
||||
}
|
||||
|
||||
return errAttemptUnavailable
|
||||
}
|
||||
|
||||
func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error {
|
||||
object, err := w.s3.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer object.Body.Close()
|
||||
|
||||
hash := sha256.New()
|
||||
n, err := io.Copy(hash, io.LimitReader(object.Body, int64(size)+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != int64(size) || hex.EncodeToString(hash.Sum(nil)) != checksum {
|
||||
return errEffectChecksumMismatch
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) deleteStagingObject(ctx context.Context, key string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
row, err := q.LockAttachmentByStagingKey(ctx, key)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.Status != db.AttachmentStatusReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := w.s3.Delete(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
"mtusi2026-messenger/internal/model"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cause error) error {
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
row, err := q.LockAttachmentJob(ctx, db.LockAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
code := "processing_failed"
|
||||
permanent := errors.Is(cause, errInvalidSource) || errors.Is(cause, s3store.ErrObjectNotFound)
|
||||
if permanent {
|
||||
code = "invalid_source"
|
||||
} else if errors.Is(cause, context.DeadlineExceeded) {
|
||||
code = "job_timeout"
|
||||
}
|
||||
|
||||
var n int64
|
||||
if permanent || row.Attempts >= w.cfg.MaxAttempts {
|
||||
if !permanent {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
|
||||
n, err = q.FailAttachmentJob(ctx, db.FailAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
ErrorCode: code,
|
||||
})
|
||||
} else {
|
||||
n, err = q.RetryAttachmentJob(ctx, db.RetryAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
ErrorCode: code,
|
||||
})
|
||||
if err == nil && n == 1 {
|
||||
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: row.ID,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return errLeaseExpired
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Warn(
|
||||
"attachment processing failed",
|
||||
"job_id",
|
||||
job.JobID,
|
||||
"attempt",
|
||||
job.Attempt,
|
||||
"error_code",
|
||||
code,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) recoverLoop(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
recovered, err := w.recoverOne(ctx)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
w.log.Warn("recover attachment lease", "error", err)
|
||||
}
|
||||
|
||||
if (err != nil || !recovered) && !wait(ctx, time.Second) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) recoverOne(ctx context.Context) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
rows, err := q.LockExpiredAttachmentJobs(ctx, 1)
|
||||
if err != nil || len(rows) == 0 {
|
||||
return false, err
|
||||
}
|
||||
|
||||
row := rows[0]
|
||||
if row.Attempts >= w.cfg.MaxAttempts {
|
||||
_, err = q.FailExpiredAttachmentJob(ctx, row.ID)
|
||||
} else {
|
||||
_, err = q.RecoverExpiredAttachmentJob(ctx, row.ID)
|
||||
if err == nil {
|
||||
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: row.ID,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
w.log.Info("attachment lease recovered", "attachment_id", row.ID, "attempt", row.Attempts)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func retryDelay(attempt int) time.Duration {
|
||||
ceiling := time.Second * time.Duration(1<<min(max(attempt, 1), 6))
|
||||
return ceiling/2 + time.Duration(rand.Int64N(int64(ceiling/2)))
|
||||
}
|
||||
@@ -2,18 +2,12 @@ package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
@@ -24,14 +18,6 @@ import (
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
var (
|
||||
errLeaseExpired = errors.New("attachment lease expired")
|
||||
errMissingThumbnail = errors.New("ready attachment has no thumbnail")
|
||||
errAttemptProcessing = errors.New("attachment attempt is still processing")
|
||||
errAttemptUnavailable = errors.New("attachment attempt is not yet available")
|
||||
errEffectChecksumMismatch = errors.New("attachment effect checksum mismatch")
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
pool *pgxpool.Pool
|
||||
q *db.Queries
|
||||
@@ -172,323 +158,6 @@ func (w *Worker) consume(ctx context.Context, client *rabbitmq.Client) error {
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error {
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
row, err := w.q.ClaimAttachmentJob(jobCtx, db.ClaimAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
LeaseUntil: time.Now().Add(w.cfg.Timeout),
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return w.duplicate(jobCtx, job)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Info("attachment processing", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
|
||||
variant, processErr := w.process(jobCtx, row)
|
||||
if processErr != nil {
|
||||
resultCtx, resultCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer resultCancel()
|
||||
|
||||
return w.finishFailure(resultCtx, job, processErr)
|
||||
}
|
||||
|
||||
tx, err := w.pool.Begin(jobCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
_, err = q.LockAttachmentJob(jobCtx, db.LockAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.CreateAttachmentVariant(jobCtx, variant); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := q.CompleteAttachmentJob(jobCtx, db.CompleteAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return errLeaseExpired
|
||||
}
|
||||
|
||||
if err := tx.Commit(jobCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.StagingKey != nil {
|
||||
err := w.deleteStagingObject(ctx, *row.StagingKey)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
w.log.Warn("delete ready attachment staging", "attachment_id", row.ID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
|
||||
row, err := w.q.GetAttachmentByID(ctx, job.AttachmentID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.JobID == nil || *row.JobID != job.JobID || row.PipelineVersion != job.PipelineVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusFailed {
|
||||
return nil
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusReady {
|
||||
variants, err := w.q.ListAttachmentVariants(ctx, db.ListAttachmentVariantsParams{
|
||||
AttachmentID: row.ID,
|
||||
PipelineVersion: row.PipelineVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, variant := range variants {
|
||||
if variant.Variant == db.AttachmentVariantNameThumbnail {
|
||||
return w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256)
|
||||
}
|
||||
}
|
||||
|
||||
return errMissingThumbnail
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusProcessing && row.Attempts == job.Attempt {
|
||||
return errAttemptProcessing
|
||||
}
|
||||
|
||||
if row.Attempts >= job.Attempt {
|
||||
return nil
|
||||
}
|
||||
|
||||
return errAttemptUnavailable
|
||||
}
|
||||
|
||||
func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error {
|
||||
object, err := w.s3.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer object.Body.Close()
|
||||
|
||||
hash := sha256.New()
|
||||
n, err := io.Copy(hash, io.LimitReader(object.Body, int64(size)+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != int64(size) || hex.EncodeToString(hash.Sum(nil)) != checksum {
|
||||
return errEffectChecksumMismatch
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cause error) error {
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
row, err := q.LockAttachmentJob(ctx, db.LockAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
code := "processing_failed"
|
||||
permanent := errors.Is(cause, errInvalidSource) || errors.Is(cause, s3store.ErrObjectNotFound)
|
||||
if permanent {
|
||||
code = "invalid_source"
|
||||
} else if errors.Is(cause, context.DeadlineExceeded) {
|
||||
code = "job_timeout"
|
||||
}
|
||||
|
||||
var n int64
|
||||
if permanent || row.Attempts >= w.cfg.MaxAttempts {
|
||||
if !permanent {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
|
||||
n, err = q.FailAttachmentJob(ctx, db.FailAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
ErrorCode: code,
|
||||
})
|
||||
} else {
|
||||
n, err = q.RetryAttachmentJob(ctx, db.RetryAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
ErrorCode: code,
|
||||
})
|
||||
if err == nil && n == 1 {
|
||||
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: row.ID,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return errLeaseExpired
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Warn(
|
||||
"attachment processing failed",
|
||||
"job_id",
|
||||
job.JobID,
|
||||
"attempt",
|
||||
job.Attempt,
|
||||
"error_code",
|
||||
code,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) deleteStagingObject(ctx context.Context, key string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
row, err := q.LockAttachmentByStagingKey(ctx, key)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.Status != db.AttachmentStatusReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := w.s3.Delete(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (w *Worker) recoverLoop(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
recovered, err := w.recoverOne(ctx)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
w.log.Warn("recover attachment lease", "error", err)
|
||||
}
|
||||
|
||||
if (err != nil || !recovered) && !wait(ctx, time.Second) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) recoverOne(ctx context.Context) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
rows, err := q.LockExpiredAttachmentJobs(ctx, 1)
|
||||
if err != nil || len(rows) == 0 {
|
||||
return false, err
|
||||
}
|
||||
|
||||
row := rows[0]
|
||||
if row.Attempts >= w.cfg.MaxAttempts {
|
||||
_, err = q.FailExpiredAttachmentJob(ctx, row.ID)
|
||||
} else {
|
||||
_, err = q.RecoverExpiredAttachmentJob(ctx, row.ID)
|
||||
if err == nil {
|
||||
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: row.ID,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
w.log.Info("attachment lease recovered", "attachment_id", row.ID, "attempt", row.Attempts)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func wait(ctx context.Context, duration time.Duration) bool {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
@@ -500,8 +169,3 @@ func wait(ctx context.Context, duration time.Duration) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func retryDelay(attempt int) time.Duration {
|
||||
ceiling := time.Second * time.Duration(1<<min(max(attempt, 1), 6))
|
||||
return ceiling/2 + time.Duration(rand.Int64N(int64(ceiling/2)))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user