Files
mtusi2026lab5/internal/worker/worker.go
T

466 lines
9.8 KiB
Go

package worker
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"log/slog"
"math/rand/v2"
"sync"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
amqp "github.com/rabbitmq/amqp091-go"
"mtusi2026-messenger/internal/config"
"mtusi2026-messenger/internal/db"
"mtusi2026-messenger/internal/model"
"mtusi2026-messenger/internal/rabbitmq"
s3store "mtusi2026-messenger/internal/s3"
)
var (
errLeaseExpired = errors.New("attachment lease expired")
errMissingThumbnail = errors.New("ready attachment has no thumbnail")
errAttemptProcessing = errors.New("attachment attempt is still processing")
errAttemptUnavailable = errors.New("attachment attempt is not yet available")
errEffectChecksumMismatch = errors.New("attachment effect checksum mismatch")
)
type Worker struct {
pool *pgxpool.Pool
q *db.Queries
s3 *s3store.Client
brokerURL string
cfg config.WorkerConfig
maxBytes, maxPixels int
decodeSlot chan struct{}
log *slog.Logger
}
func New(
pool *pgxpool.Pool,
s3 *s3store.Client,
brokerURL string,
cfg config.WorkerConfig,
maxBytes, maxPixels int,
log *slog.Logger,
) *Worker {
return &Worker{
pool: pool,
q: db.New(pool),
s3: s3,
brokerURL: brokerURL,
cfg: cfg,
maxBytes: maxBytes,
maxPixels: maxPixels,
decodeSlot: make(chan struct{}, 1),
log: log,
}
}
func (w *Worker) Run(ctx context.Context) {
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
w.recoverLoop(ctx)
}()
for range w.cfg.Concurrency {
wg.Add(1)
go func() {
defer wg.Done()
w.consumeLoop(ctx)
}()
}
wg.Wait()
}
func (w *Worker) consumeLoop(ctx context.Context) {
for ctx.Err() == nil {
client, err := rabbitmq.New(w.brokerURL)
if err == nil {
err = w.consume(ctx, client)
_ = client.Close()
}
if ctx.Err() != nil {
return
}
w.log.Warn("worker consumer reconnect", "error", err)
if !wait(ctx, 2*time.Second) {
return
}
}
}
func (w *Worker) consume(ctx context.Context, client *rabbitmq.Client) error {
stop := context.AfterFunc(ctx, func() {
_ = client.Close()
})
defer stop()
deliveries, err := client.Consume(ctx, 1)
if err != nil {
return err
}
for {
select {
case <-ctx.Done():
return ctx.Err()
case delivery, ok := <-deliveries:
if !ok {
return amqp.ErrClosed
}
var job model.AttachmentJob
if json.Unmarshal(delivery.Body, &job) != nil ||
job.SchemaVersion != 1 ||
job.Type != "attachment.process.v1" ||
job.JobID == uuid.Nil ||
job.AttachmentID == uuid.Nil ||
job.PipelineVersion != 1 ||
job.Attempt < 1 ||
job.CreatedAt.IsZero() {
if err := delivery.Reject(false); err != nil {
return err
}
w.log.Warn("invalid attachment job rejected")
continue
}
err := w.handle(ctx, job)
if err == nil {
if err := delivery.Ack(false); err != nil {
return err
}
} else {
if ctx.Err() != nil {
return ctx.Err()
}
w.log.Warn(
"attachment job not acknowledged",
"job_id",
job.JobID,
"attempt",
job.Attempt,
"error",
err,
)
if !wait(ctx, time.Second) {
return ctx.Err()
}
if err := delivery.Nack(false, true); err != nil {
return err
}
}
}
}
}
func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error {
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.Timeout)
defer cancel()
row, err := w.q.ClaimAttachmentJob(jobCtx, db.ClaimAttachmentJobParams{
ID: job.AttachmentID,
JobID: job.JobID,
PipelineVersion: job.PipelineVersion,
Attempt: job.Attempt,
LeaseUntil: time.Now().Add(w.cfg.Timeout),
})
if errors.Is(err, pgx.ErrNoRows) {
return w.duplicate(jobCtx, job)
}
if err != nil {
return err
}
w.log.Info("attachment processing", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
variant, processErr := w.process(jobCtx, row)
if processErr != nil {
resultCtx, resultCancel := context.WithTimeout(ctx, 5*time.Second)
defer resultCancel()
return w.finishFailure(resultCtx, job, processErr)
}
tx, err := w.pool.Begin(jobCtx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
q := w.q.WithTx(tx)
_, err = q.LockAttachmentJob(jobCtx, db.LockAttachmentJobParams{
ID: job.AttachmentID,
JobID: job.JobID,
PipelineVersion: job.PipelineVersion,
Attempt: job.Attempt,
})
if err != nil {
return err
}
if err := q.CreateAttachmentVariant(jobCtx, variant); err != nil {
return err
}
n, err := q.CompleteAttachmentJob(jobCtx, db.CompleteAttachmentJobParams{
ID: row.ID,
Attempt: job.Attempt,
})
if err != nil {
return err
}
if n != 1 {
return errLeaseExpired
}
if err := tx.Commit(jobCtx); err != nil {
return err
}
w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
return nil
}
func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
row, err := w.q.GetAttachmentByID(ctx, job.AttachmentID)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
if row.JobID == nil || *row.JobID != job.JobID || row.PipelineVersion != job.PipelineVersion {
return nil
}
if row.Status == db.AttachmentStatusFailed {
return nil
}
if row.Status == db.AttachmentStatusReady {
variants, err := w.q.ListAttachmentVariants(ctx, db.ListAttachmentVariantsParams{
AttachmentID: row.ID,
PipelineVersion: row.PipelineVersion,
})
if err != nil {
return err
}
if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
return err
}
for _, variant := range variants {
if variant.Variant == db.AttachmentVariantNameThumbnail {
return w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256)
}
}
return errMissingThumbnail
}
if row.Status == db.AttachmentStatusProcessing && row.Attempts == job.Attempt {
return errAttemptProcessing
}
if row.Attempts >= job.Attempt {
return nil
}
return errAttemptUnavailable
}
func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error {
object, err := w.s3.Get(ctx, key)
if err != nil {
return err
}
defer object.Body.Close()
hash := sha256.New()
n, err := io.Copy(hash, io.LimitReader(object.Body, int64(size)+1))
if err != nil {
return err
}
if n != int64(size) || hex.EncodeToString(hash.Sum(nil)) != checksum {
return errEffectChecksumMismatch
}
return nil
}
func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cause error) error {
tx, err := w.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
q := w.q.WithTx(tx)
row, err := q.LockAttachmentJob(ctx, db.LockAttachmentJobParams{
ID: job.AttachmentID,
JobID: job.JobID,
PipelineVersion: job.PipelineVersion,
Attempt: job.Attempt,
})
if err != nil {
return err
}
code := "processing_failed"
permanent := errors.Is(cause, errInvalidSource) || errors.Is(cause, s3store.ErrObjectNotFound)
if permanent {
code = "invalid_source"
} else if errors.Is(cause, context.DeadlineExceeded) {
code = "job_timeout"
}
var n int64
if permanent || row.Attempts >= w.cfg.MaxAttempts {
if !permanent {
code = "retry_exhausted"
}
n, err = q.FailAttachmentJob(ctx, db.FailAttachmentJobParams{
ID: row.ID,
Attempt: job.Attempt,
ErrorCode: code,
})
} else {
n, err = q.RetryAttachmentJob(ctx, db.RetryAttachmentJobParams{
ID: row.ID,
Attempt: job.Attempt,
ErrorCode: code,
})
if err == nil && n == 1 {
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
AttachmentID: row.ID,
Attempt: row.Attempts + 1,
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
})
}
}
if err != nil {
return err
}
if n != 1 {
return errLeaseExpired
}
if err := tx.Commit(ctx); err != nil {
return err
}
w.log.Warn(
"attachment processing failed",
"job_id",
job.JobID,
"attempt",
job.Attempt,
"error_code",
code,
)
return nil
}
func (w *Worker) recoverLoop(ctx context.Context) {
for ctx.Err() == nil {
recovered, err := w.recoverOne(ctx)
if err != nil && ctx.Err() == nil {
w.log.Warn("recover attachment lease", "error", err)
}
if (err != nil || !recovered) && !wait(ctx, time.Second) {
return
}
}
}
func (w *Worker) recoverOne(ctx context.Context) (bool, error) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tx, err := w.pool.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
q := w.q.WithTx(tx)
rows, err := q.LockExpiredAttachmentJobs(ctx, 1)
if err != nil || len(rows) == 0 {
return false, err
}
row := rows[0]
if row.Attempts >= w.cfg.MaxAttempts {
_, err = q.FailExpiredAttachmentJob(ctx, row.ID)
} else {
_, err = q.RecoverExpiredAttachmentJob(ctx, row.ID)
if err == nil {
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
AttachmentID: row.ID,
Attempt: row.Attempts + 1,
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
})
}
}
if err != nil {
return false, err
}
if err := tx.Commit(ctx); err != nil {
return false, err
}
w.log.Info("attachment lease recovered", "attachment_id", row.ID, "attempt", row.Attempts)
return true, nil
}
func wait(ctx context.Context, duration time.Duration) bool {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
func retryDelay(attempt int) time.Duration {
ceiling := time.Second * time.Duration(1<<min(max(attempt, 1), 6))
return ceiling/2 + time.Duration(rand.Int64N(int64(ceiling/2)))
}