Files
mtusi2026lab6/internal/postgres/attachment_recovery.go
T

211 lines
6.8 KiB
Go

package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
)
func (s *Store) RecoverExpiredAttachmentJobs(
ctx context.Context,
limit int,
maxAttempts int,
retryAt time.Time,
) (model.RecoveryResult, error) {
if limit < 1 || limit > 1000 || maxAttempts < 1 {
return model.RecoveryResult{}, model.ErrInvalidInput
}
if retryAt.IsZero() {
retryAt = time.Now().UTC()
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
const selectExpired = `
SELECT id
FROM attachment_jobs
WHERE status = 'processing' AND lease_until <= clock_timestamp()
ORDER BY lease_until, id
LIMIT $1
FOR UPDATE SKIP LOCKED`
rows, err := tx.Query(ctx, selectExpired, limit)
if err != nil {
return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err)
}
jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err)
}
var recovered model.RecoveryResult
for _, jobID := range jobIDs {
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
if err != nil {
return model.RecoveryResult{}, err
}
if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) {
continue
}
if job.PipelineVersion != attachment.PipelineVersion {
if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil {
return model.RecoveryResult{}, err
}
recovered.Failed++
continue
}
if job.Attempt >= maxAttempts {
if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil {
return model.RecoveryResult{}, err
}
recovered.Failed++
continue
}
job.Attempt++
job.Status = model.AttachmentJobQueued
job.AvailableAt = retryAt.UTC()
const requeueJob = `
UPDATE attachment_jobs
SET status = 'queued', attempt = $2, available_at = $3,
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = 'lease_expired', updated_at = clock_timestamp()
WHERE id = $1`
if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err)
}
const requeueAttachment = `
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
return model.RecoveryResult{}, err
}
recovered.Requeued++
}
if err := tx.Commit(ctx); err != nil {
return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err)
}
return recovered, nil
}
func markExpiredJobFailed(
ctx context.Context,
tx pgx.Tx,
job model.AttachmentJob,
attachment model.Attachment,
errorCode string,
failAttachment bool,
) error {
const failJob = `
UPDATE attachment_jobs
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = $2, updated_at = clock_timestamp()
WHERE id = $1`
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
return fmt.Errorf("fail expired attachment job: %w", err)
}
if !failAttachment {
return nil
}
const failAttachmentQuery = `
UPDATE attachments
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil {
return fmt.Errorf("fail attachment after expired lease: %w", err)
}
return nil
}
func (s *Store) ReplayFailedAttachment(
ctx context.Context,
attachmentID string,
actor string,
reason string,
) (model.AttachmentJob, error) {
if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 {
return model.AttachmentJob{}, model.ErrInvalidInput
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
const selectJob = `
SELECT j.id
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed'
FOR UPDATE OF j, a`
var jobID string
if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.AttachmentJob{}, ErrConflict
}
return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err)
}
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
if err != nil {
return model.AttachmentJob{}, err
}
previousAttempt := job.Attempt
job.Attempt++
job.Status = model.AttachmentJobQueued
job.AvailableAt = time.Now().UTC()
job.ErrorCode = nil
job.LeaseToken = ""
job.LeaseOwner = ""
job.LeaseUntil = nil
const replayJob = `
UPDATE attachment_jobs
SET status = 'queued', attempt = $2, available_at = $3,
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1`
if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err)
}
const replayAttachment = `
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = $1 AND pipeline_version = $2`
if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil {
return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err)
}
const insertAudit = `
INSERT INTO attachment_replay_audit (
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason
)
VALUES ($1, $2, $3, $4, $5, $6, $7)`
if _, err := tx.Exec(
ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID,
previousAttempt, job.Attempt, actor, reason,
); err != nil {
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
}
eventJob := job
eventJob.CreatedAt = time.Now().UTC()
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
return model.AttachmentJob{}, err
}
if err := tx.Commit(ctx); err != nil {
return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err)
}
return job, nil
}