138 lines
4.4 KiB
Go
138 lines
4.4 KiB
Go
package messenger
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/minio/minio-go/v7"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Operator-only CLI. It exposes no unauthenticated HTTP mutation endpoint.
|
|
func (p *Pipeline) Replay(ctx context.Context, id string, newVersion bool) error {
|
|
if !uuidRE.MatchString(id) {
|
|
return fmt.Errorf("expected attachment UUID")
|
|
}
|
|
tx, e := p.DB.Begin(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
var status, key, stage string
|
|
var version int
|
|
e = tx.QueryRow(ctx, `SELECT status,object_key,COALESCE(staging_key,''),pipeline_version FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&status, &key, &stage, &version)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if newVersion {
|
|
if status != "ready" && status != "queued" && status != "processing" {
|
|
return fmt.Errorf("replay failed before version upgrade")
|
|
}
|
|
if version >= 2 {
|
|
return fmt.Errorf("supported pipeline versions: 1 and 2")
|
|
}
|
|
version++
|
|
if status == "ready" {
|
|
stage = key
|
|
}
|
|
} else {
|
|
if status != "failed" {
|
|
return fmt.Errorf("only failed can be replayed")
|
|
}
|
|
}
|
|
if _, e = p.Objects.Client.StatObject(ctx, p.Objects.Bucket, stage, minio.StatObjectOptions{}); e != nil {
|
|
return fmt.Errorf("source unavailable; restore before replay")
|
|
}
|
|
var jobID string
|
|
if newVersion {
|
|
jobID = uuid()
|
|
if _, e = tx.Exec(ctx, "UPDATE pipeline_jobs SET finished_at=clock_timestamp() WHERE attachment_id=$1 AND finished_at IS NULL", id); e != nil {
|
|
return e
|
|
}
|
|
_, e = tx.Exec(ctx, `INSERT INTO pipeline_jobs(job_id,attachment_id,pipeline_version) VALUES($1,$2,$3)`, jobID, id, version)
|
|
} else {
|
|
e = tx.QueryRow(ctx, `UPDATE pipeline_jobs SET attempt=0,lease_token=NULL,lease_until=NULL,published_at=NULL,finished_at=NULL,next_attempt_at=clock_timestamp() WHERE attachment_id=$1 AND pipeline_version=$2 RETURNING job_id::text`, id, version).Scan(&jobID)
|
|
}
|
|
if e != nil {
|
|
return e
|
|
}
|
|
_, e = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,variants='[]',pipeline_version=$2,staging_key=$3,updated_at=clock_timestamp() WHERE id=$1`, id, version, stage)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
event := "operator_replay"
|
|
if newVersion {
|
|
event = "operator_new_version"
|
|
}
|
|
if _, e = tx.Exec(ctx, `INSERT INTO pipeline_audit(job_id,event,attempt) VALUES($1,$2,0)`, jobID, event); e != nil {
|
|
return e
|
|
}
|
|
return tx.Commit(ctx)
|
|
}
|
|
|
|
// Only staging is collected automatically. Live queued/processing work and final
|
|
// objects are always protected. Ready stage is redundant; failed source has 7d
|
|
// retention. Unattributed aborted uploads/intents have a conservative 24h grace.
|
|
func (p *Pipeline) Cleanup(ctx context.Context, apply bool) error {
|
|
for object := range p.Objects.Client.ListObjects(ctx, p.Objects.Bucket, minio.ListObjectsOptions{Prefix: "staging/", Recursive: true}) {
|
|
if object.Err != nil {
|
|
return object.Err
|
|
}
|
|
if time.Since(object.LastModified) < 24*time.Hour {
|
|
continue
|
|
}
|
|
tx, e := p.DB.Begin(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if _, e = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,4))", object.Key); e != nil {
|
|
tx.Rollback(ctx)
|
|
return e
|
|
}
|
|
var status string
|
|
var updated time.Time
|
|
e = tx.QueryRow(ctx, `SELECT status,updated_at FROM attachments WHERE staging_key=$1 FOR UPDATE`, object.Key).Scan(&status, &updated)
|
|
if e != nil && !errors.Is(e, pgx.ErrNoRows) {
|
|
tx.Rollback(ctx)
|
|
return e
|
|
}
|
|
safe := errors.Is(e, pgx.ErrNoRows) || status == "ready" || (status == "failed" && time.Since(updated) > 7*24*time.Hour)
|
|
if safe {
|
|
p.Log.Info("staging_cleanup", "event", "staging_cleanup", "apply", apply, "status", status)
|
|
if apply {
|
|
if e = p.Objects.Client.RemoveObject(ctx, p.Objects.Bucket, object.Key, minio.RemoveObjectOptions{}); e != nil {
|
|
tx.Rollback(ctx)
|
|
return e
|
|
}
|
|
if _, e = tx.Exec(ctx, `DELETE FROM upload_intents WHERE object_key=$1`, object.Key); e != nil {
|
|
tx.Rollback(ctx)
|
|
return e
|
|
}
|
|
}
|
|
}
|
|
if e = tx.Commit(ctx); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func (p *Pipeline) RunHousekeeping(ctx context.Context) error {
|
|
for ctx.Err() == nil {
|
|
if e := p.Cleanup(ctx, true); e != nil {
|
|
p.Log.Warn("cleanup_retry", "event", "cleanup_retry")
|
|
}
|
|
if !wait(ctx, time.Hour) {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func ParseOperatorArgs(args []string) (string, bool, error) {
|
|
if len(args) != 1 || strings.TrimSpace(args[0]) == "" {
|
|
return "", false, fmt.Errorf("expected one attachment UUID")
|
|
}
|
|
return args[0], false, nil
|
|
}
|