forked from mtusi2026/mtusi2026lab5
151 lines
3.3 KiB
Go
151 lines
3.3 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"math/rand/v2"
|
|
"time"
|
|
|
|
"mtusi2026-messenger/internal/db"
|
|
"mtusi2026-messenger/internal/model"
|
|
s3store "mtusi2026-messenger/internal/s3"
|
|
)
|
|
|
|
func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cause error) error {
|
|
tx, err := w.pool.Begin(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer tx.Rollback(context.Background())
|
|
|
|
q := w.q.WithTx(tx)
|
|
row, err := q.LockAttachmentJob(ctx, db.LockAttachmentJobParams{
|
|
ID: job.AttachmentID,
|
|
JobID: job.JobID,
|
|
PipelineVersion: job.PipelineVersion,
|
|
Attempt: job.Attempt,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
code := "processing_failed"
|
|
permanent := errors.Is(cause, ErrInvalidSource) || errors.Is(cause, s3store.ErrObjectNotFound)
|
|
if permanent {
|
|
code = "invalid_source"
|
|
} else if errors.Is(cause, context.DeadlineExceeded) {
|
|
code = "job_timeout"
|
|
}
|
|
|
|
var n int64
|
|
if permanent || row.Attempts-row.RetryBase >= w.cfg.MaxAttempts {
|
|
if !permanent {
|
|
code = "retry_exhausted"
|
|
}
|
|
|
|
n, err = q.FailAttachmentJob(ctx, db.FailAttachmentJobParams{
|
|
ID: row.ID,
|
|
Attempt: job.Attempt,
|
|
ErrorCode: code,
|
|
})
|
|
} else {
|
|
n, err = q.RetryAttachmentJob(ctx, db.RetryAttachmentJobParams{
|
|
ID: row.ID,
|
|
Attempt: job.Attempt,
|
|
ErrorCode: code,
|
|
})
|
|
if err == nil && n == 1 {
|
|
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
|
AttachmentID: row.ID,
|
|
Attempt: row.Attempts + 1,
|
|
AvailableAt: time.Now().Add(retryDelay(row.Attempts - row.RetryBase)),
|
|
})
|
|
}
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if n != 1 {
|
|
return errLeaseExpired
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
w.log.Warn(
|
|
"attachment processing failed",
|
|
"job_id",
|
|
job.JobID,
|
|
"attempt",
|
|
job.Attempt,
|
|
"error_code",
|
|
code,
|
|
)
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) recoverLoop(ctx context.Context) {
|
|
for ctx.Err() == nil {
|
|
recovered, err := w.recoverOne(ctx)
|
|
if err != nil && ctx.Err() == nil {
|
|
w.log.Warn("recover attachment lease", "error", err)
|
|
}
|
|
|
|
if (err != nil || !recovered) && !wait(ctx, time.Second) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *Worker) recoverOne(ctx context.Context) (bool, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
|
|
tx, err := w.pool.Begin(ctx)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
defer tx.Rollback(context.Background())
|
|
|
|
q := w.q.WithTx(tx)
|
|
rows, err := q.LockExpiredAttachmentJobs(ctx, 1)
|
|
if err != nil || len(rows) == 0 {
|
|
return false, err
|
|
}
|
|
|
|
row := rows[0]
|
|
if row.Attempts-row.RetryBase >= w.cfg.MaxAttempts {
|
|
_, err = q.FailExpiredAttachmentJob(ctx, row.ID)
|
|
} else {
|
|
_, err = q.RecoverExpiredAttachmentJob(ctx, row.ID)
|
|
if err == nil {
|
|
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
|
AttachmentID: row.ID,
|
|
Attempt: row.Attempts + 1,
|
|
AvailableAt: time.Now().Add(retryDelay(row.Attempts - row.RetryBase)),
|
|
})
|
|
}
|
|
}
|
|
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return false, err
|
|
}
|
|
|
|
w.log.Info("attachment lease recovered", "attachment_id", row.ID, "attempt", row.Attempts)
|
|
return true, nil
|
|
}
|
|
|
|
func retryDelay(attempt int) time.Duration {
|
|
ceiling := time.Second * time.Duration(1<<min(max(attempt, 1), 6))
|
|
return ceiling/2 + time.Duration(rand.Int64N(int64(ceiling/2)))
|
|
}
|