136 lines
4.5 KiB
Go
136 lines
4.5 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"messenger/internal/attachment"
|
|
"messenger/internal/platform/apperr"
|
|
)
|
|
|
|
type FailedJob struct {
|
|
JobID uuid.UUID `json:"job_id"`
|
|
AttachmentID uuid.UUID `json:"attachment_id"`
|
|
PipelineVersion int `json:"pipeline_version"`
|
|
Attempt int `json:"attempt"`
|
|
ErrorCode string `json:"error_code"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
func (s *Store) Failed(ctx context.Context) ([]FailedJob, error) {
|
|
rows, err := s.pool.Query(ctx, `SELECT job_id,attachment_id,pipeline_version,attempt,error_code,updated_at
|
|
FROM attachment_jobs WHERE state='failed' ORDER BY updated_at,job_id`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return pgx.CollectRows(rows, pgx.RowToStructByPos[FailedJob])
|
|
}
|
|
|
|
// Lock the attachment before checking the current target, its source or an S3
|
|
// object. This serializes replay and version changes with source collection.
|
|
func lockTarget(ctx context.Context, tx pgx.Tx, id uuid.UUID) (int, []byte, error) {
|
|
var version int
|
|
var hash []byte
|
|
err := tx.QueryRow(ctx, `SELECT target_version,file_hash FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&version, &hash)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
err = apperr.NotFound("attachment not found")
|
|
}
|
|
return version, hash, err
|
|
}
|
|
|
|
func availableSource(ctx context.Context, preferred, fallback string, exists func(context.Context, string) (bool, error)) (string, error) {
|
|
ok, err := exists(ctx, preferred)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if ok {
|
|
return preferred, nil
|
|
}
|
|
if fallback != preferred {
|
|
ok, err = exists(ctx, fallback)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if ok {
|
|
return fallback, nil
|
|
}
|
|
}
|
|
return "", apperr.Conflict("processing source is missing")
|
|
}
|
|
|
|
func (s *Store) Reprocess(ctx context.Context, id uuid.UUID, version int, exists func(context.Context, string) (bool, error)) error {
|
|
if version != 1 && version != 2 {
|
|
return apperr.Invalid("unsupported pipeline version")
|
|
}
|
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
current, hash, err := lockTarget(ctx, tx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if version <= current {
|
|
return apperr.Conflict("target version must increase")
|
|
}
|
|
var source string
|
|
if err := tx.QueryRow(ctx, `SELECT source_key FROM attachment_jobs
|
|
WHERE attachment_id=$1 AND pipeline_version=$2 FOR UPDATE`, id, current).Scan(&source); err != nil {
|
|
return err
|
|
}
|
|
source, err = availableSource(ctx, attachment.BlobKey(hash), source, exists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Keep committed result rows immutable and readable until the switch.
|
|
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='superseded',lease_token=NULL,lease_until=NULL,updated_at=clock_timestamp()
|
|
WHERE attachment_id=$1 AND pipeline_version=$2 AND state<>'ready'`, id, current); err != nil {
|
|
return err
|
|
}
|
|
if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key)
|
|
VALUES ($1,$2,$3)`, id, version, source); err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.Exec(ctx, `UPDATE attachments SET target_version=$2 WHERE id=$1`, id, version)
|
|
return err
|
|
})
|
|
}
|
|
|
|
func (s *Store) Replay(ctx context.Context, id uuid.UUID, actor string, exists func(context.Context, string) (bool, error)) error {
|
|
actor = strings.TrimSpace(actor)
|
|
if actor == "" || len(actor) > 128 {
|
|
return apperr.Invalid("replay requires an operator name of up to 128 bytes")
|
|
}
|
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
current, hash, err := lockTarget(ctx, tx, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var jobID uuid.UUID
|
|
var source, code string
|
|
var attempt int
|
|
err = tx.QueryRow(ctx, `SELECT job_id,source_key,error_code,attempt FROM attachment_jobs
|
|
WHERE attachment_id=$1 AND pipeline_version=$2 AND state='failed' FOR UPDATE`, id, current).
|
|
Scan(&jobID, &source, &code, &attempt)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return apperr.Conflict("current job is not failed")
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
source, err = availableSource(ctx, source, attachment.BlobKey(hash), exists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err = tx.Exec(ctx, `INSERT INTO attachment_replays(job_id,attempt,actor,previous_error) VALUES ($1,$2,$3,$4)`,
|
|
jobID, attempt+1, actor, code); err != nil {
|
|
return err
|
|
}
|
|
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET state='queued',attempt=attempt+1,budget_attempt=1,source_key=$2,
|
|
published_at=NULL,available_at=clock_timestamp(),updated_at=clock_timestamp(),error_code=NULL WHERE job_id=$1`, jobID, source)
|
|
return err
|
|
})
|
|
}
|