forked from mtusi2026/mtusi2026lab6
211 lines
6.8 KiB
Go
211 lines
6.8 KiB
Go
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
|
)
|
|
|
|
func (s *Store) RecoverExpiredAttachmentJobs(
|
|
ctx context.Context,
|
|
limit int,
|
|
maxAttempts int,
|
|
retryAt time.Time,
|
|
) (model.RecoveryResult, error) {
|
|
if limit < 1 || limit > 1000 || maxAttempts < 1 {
|
|
return model.RecoveryResult{}, model.ErrInvalidInput
|
|
}
|
|
if retryAt.IsZero() {
|
|
retryAt = time.Now().UTC()
|
|
}
|
|
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
|
|
|
const selectExpired = `
|
|
SELECT id
|
|
FROM attachment_jobs
|
|
WHERE status = 'processing' AND lease_until <= clock_timestamp()
|
|
ORDER BY lease_until, id
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED`
|
|
rows, err := tx.Query(ctx, selectExpired, limit)
|
|
if err != nil {
|
|
return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err)
|
|
}
|
|
jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
|
if err != nil {
|
|
return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err)
|
|
}
|
|
|
|
var recovered model.RecoveryResult
|
|
for _, jobID := range jobIDs {
|
|
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
|
if err != nil {
|
|
return model.RecoveryResult{}, err
|
|
}
|
|
if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) {
|
|
continue
|
|
}
|
|
if job.PipelineVersion != attachment.PipelineVersion {
|
|
if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil {
|
|
return model.RecoveryResult{}, err
|
|
}
|
|
recovered.Failed++
|
|
continue
|
|
}
|
|
if job.Attempt >= maxAttempts {
|
|
if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil {
|
|
return model.RecoveryResult{}, err
|
|
}
|
|
recovered.Failed++
|
|
continue
|
|
}
|
|
|
|
job.Attempt++
|
|
job.Status = model.AttachmentJobQueued
|
|
job.AvailableAt = retryAt.UTC()
|
|
const requeueJob = `
|
|
UPDATE attachment_jobs
|
|
SET status = 'queued', attempt = $2, available_at = $3,
|
|
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
|
error_code = 'lease_expired', updated_at = clock_timestamp()
|
|
WHERE id = $1`
|
|
if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
|
|
return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err)
|
|
}
|
|
const requeueAttachment = `
|
|
UPDATE attachments
|
|
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
|
WHERE id = $1 AND pipeline_version = $2`
|
|
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
|
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
|
|
}
|
|
eventJob := job
|
|
eventJob.CreatedAt = time.Now().UTC()
|
|
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
|
return model.RecoveryResult{}, err
|
|
}
|
|
recovered.Requeued++
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err)
|
|
}
|
|
return recovered, nil
|
|
}
|
|
|
|
func markExpiredJobFailed(
|
|
ctx context.Context,
|
|
tx pgx.Tx,
|
|
job model.AttachmentJob,
|
|
attachment model.Attachment,
|
|
errorCode string,
|
|
failAttachment bool,
|
|
) error {
|
|
const failJob = `
|
|
UPDATE attachment_jobs
|
|
SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
|
error_code = $2, updated_at = clock_timestamp()
|
|
WHERE id = $1`
|
|
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
|
|
return fmt.Errorf("fail expired attachment job: %w", err)
|
|
}
|
|
if !failAttachment {
|
|
return nil
|
|
}
|
|
const failAttachmentQuery = `
|
|
UPDATE attachments
|
|
SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp()
|
|
WHERE id = $1 AND pipeline_version = $2`
|
|
if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil {
|
|
return fmt.Errorf("fail attachment after expired lease: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) ReplayFailedAttachment(
|
|
ctx context.Context,
|
|
attachmentID string,
|
|
actor string,
|
|
reason string,
|
|
) (model.AttachmentJob, error) {
|
|
if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 {
|
|
return model.AttachmentJob{}, model.ErrInvalidInput
|
|
}
|
|
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
|
|
|
const selectJob = `
|
|
SELECT j.id
|
|
FROM attachment_jobs j
|
|
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
|
WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed'
|
|
FOR UPDATE OF j, a`
|
|
var jobID string
|
|
if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return model.AttachmentJob{}, ErrConflict
|
|
}
|
|
return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err)
|
|
}
|
|
job, attachment, err := lockAttachmentJob(ctx, tx, jobID)
|
|
if err != nil {
|
|
return model.AttachmentJob{}, err
|
|
}
|
|
previousAttempt := job.Attempt
|
|
job.Attempt++
|
|
job.Status = model.AttachmentJobQueued
|
|
job.AvailableAt = time.Now().UTC()
|
|
job.ErrorCode = nil
|
|
job.LeaseToken = ""
|
|
job.LeaseOwner = ""
|
|
job.LeaseUntil = nil
|
|
const replayJob = `
|
|
UPDATE attachment_jobs
|
|
SET status = 'queued', attempt = $2, available_at = $3,
|
|
lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
|
error_code = NULL, updated_at = clock_timestamp()
|
|
WHERE id = $1`
|
|
if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil {
|
|
return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err)
|
|
}
|
|
const replayAttachment = `
|
|
UPDATE attachments
|
|
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
|
|
WHERE id = $1 AND pipeline_version = $2`
|
|
if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
|
return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err)
|
|
}
|
|
const insertAudit = `
|
|
INSERT INTO attachment_replay_audit (
|
|
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason
|
|
)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7)`
|
|
if _, err := tx.Exec(
|
|
ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID,
|
|
previousAttempt, job.Attempt, actor, reason,
|
|
); err != nil {
|
|
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
|
|
}
|
|
eventJob := job
|
|
eventJob.CreatedAt = time.Now().UTC()
|
|
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
|
return model.AttachmentJob{}, err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err)
|
|
}
|
|
return job, nil
|
|
}
|