617 lines
25 KiB
Go
617 lines
25 KiB
Go
package postgres
|
||
|
||
import (
|
||
"github.com/charawein/mtusi2026lab6/internal/telemetry"
|
||
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/google/uuid"
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"github.com/charawein/mtusi2026lab6/internal/domain"
|
||
"github.com/charawein/mtusi2026lab6/internal/repository"
|
||
)
|
||
|
||
// PipelineRepository — задания, outbox и фиксация результатов обработки.
|
||
//
|
||
// Все переходы job выполняются в транзакциях с блокировкой строки job (FOR UPDATE),
|
||
// а фиксация результата защищена fencing-токеном (lease_owner + attempt): worker,
|
||
// потерявший lease, не может опубликовать результат поверх новой попытки.
|
||
type PipelineRepository struct {
|
||
pool *pgxpool.Pool
|
||
}
|
||
|
||
func NewPipelineRepository(pool *pgxpool.Pool) *PipelineRepository {
|
||
return &PipelineRepository{pool: pool}
|
||
}
|
||
|
||
// JobRow — состояние логической работы.
|
||
type JobRow struct {
|
||
JobID string
|
||
AttachmentID string
|
||
PipelineVersion int
|
||
State string
|
||
Attempt int
|
||
MaxAttempts int
|
||
NextAttemptAt time.Time
|
||
LeaseUntil *time.Time
|
||
LeaseOwner string
|
||
LastError string
|
||
CreatedAt time.Time // приём работы (durable acceptance) — для time_to_ready
|
||
}
|
||
|
||
func jobPayload(jobID, attachmentID string, version, attempt int, created time.Time) ([]byte, error) {
|
||
return json.Marshal(domain.Job{
|
||
SchemaVersion: 1, JobID: jobID, Type: domain.JobType, AttachmentID: attachmentID,
|
||
PipelineVersion: version, Attempt: attempt,
|
||
CreatedAt: created.UTC().Truncate(time.Second).Format(time.RFC3339),
|
||
})
|
||
}
|
||
|
||
func event(ctx context.Context, tx pgx.Tx, jobID, ev, detail string) error {
|
||
_, err := tx.Exec(ctx, `INSERT INTO job_events (job_id, event, detail) VALUES ($1::uuid, $2, NULLIF($3, ''))`, jobID, ev, detail)
|
||
return err
|
||
}
|
||
|
||
func outboxTx(ctx context.Context, tx pgx.Tx, jobID, attachmentID string, version, attempt int, available time.Time) error {
|
||
payload, err := jobPayload(jobID, attachmentID, version, attempt, time.Now())
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// Контекст трассы: активный span (upload, попытка worker) или, если его нет
|
||
// (sweeper), исходный контекст работы из jobs.
|
||
_, err = tx.Exec(ctx, `
|
||
INSERT INTO outbox (job_id, payload, available_at, trace_context)
|
||
VALUES ($1::uuid, $2, $3, COALESCE($4::jsonb, (SELECT trace_context FROM jobs WHERE job_id = $1::uuid)))`,
|
||
jobID, payload, available, traceJSON(ctx))
|
||
if err != nil {
|
||
return fmt.Errorf("insert outbox: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// EnqueueJobTx создаёт логическую работу (attachment, version) и её первое
|
||
// сообщение в outbox. Если работа этой версии уже есть — ничего не создаёт и
|
||
// возвращает существующий job_id (ключ эффекта не дублируется).
|
||
func EnqueueJobTx(ctx context.Context, tx pgx.Tx, attachmentID string, version, maxAttempts int, reason string) (string, error) {
|
||
jobID := uuid.NewString()
|
||
var got string
|
||
err := tx.QueryRow(ctx, `
|
||
INSERT INTO jobs (job_id, attachment_id, pipeline_version, state, max_attempts, trace_context)
|
||
VALUES ($1::uuid, $2::uuid, $3, 'queued', $4, $5::jsonb)
|
||
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
|
||
RETURNING job_id::text`, jobID, attachmentID, version, maxAttempts, traceJSON(ctx)).Scan(&got)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
err = tx.QueryRow(ctx, `SELECT job_id::text FROM jobs WHERE attachment_id = $1::uuid AND pipeline_version = $2`,
|
||
attachmentID, version).Scan(&got)
|
||
return got, err
|
||
}
|
||
if err != nil {
|
||
return "", fmt.Errorf("insert job: %w", err)
|
||
}
|
||
if err := outboxTx(ctx, tx, got, attachmentID, version, 1, time.Now()); err != nil {
|
||
return "", err
|
||
}
|
||
return got, event(ctx, tx, got, "enqueued", reason)
|
||
}
|
||
|
||
func lockJob(ctx context.Context, tx pgx.Tx, jobID string) (*JobRow, error) {
|
||
var j JobRow
|
||
var owner, lastErr *string
|
||
err := tx.QueryRow(ctx, `
|
||
SELECT job_id::text, attachment_id::text, pipeline_version, state, attempt, max_attempts,
|
||
next_attempt_at, lease_until, lease_owner, last_error, created_at
|
||
FROM jobs WHERE job_id = $1::uuid FOR UPDATE`, jobID).
|
||
Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.State, &j.Attempt, &j.MaxAttempts,
|
||
&j.NextAttemptAt, &j.LeaseUntil, &owner, &lastErr, &j.CreatedAt)
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return nil, repository.ErrNotFound
|
||
}
|
||
if err != nil {
|
||
return nil, fmt.Errorf("lock job: %w", err)
|
||
}
|
||
if owner != nil {
|
||
j.LeaseOwner = *owner
|
||
}
|
||
if lastErr != nil {
|
||
j.LastError = *lastErr
|
||
}
|
||
return &j, nil
|
||
}
|
||
|
||
// ClaimOutcome — решение worker по доставленному сообщению.
|
||
type ClaimOutcome int
|
||
|
||
const (
|
||
Claimed ClaimOutcome = iota // аренда получена — обрабатывать
|
||
AlreadyDone // эффект уже опубликован — ack без работы
|
||
Busy // валидная аренда у другого worker — ack (дубль)
|
||
NotDue // ранняя доставка до next_attempt_at — ack, outbox доставит позже
|
||
Terminal // failed / задания нет — ack
|
||
ExhaustedNow // попытки исчерпаны при захвате — уже помечено failed
|
||
)
|
||
|
||
// Claim захватывает работу: queued (и уже наступило next_attempt_at) или
|
||
// processing с истёкшим lease. Номер попытки растёт при каждом захвате.
|
||
func (r *PipelineRepository) Claim(ctx context.Context, jobID, owner string, lease time.Duration) (ClaimOutcome, *JobRow, *domain.Attachment, error) {
|
||
tx, err := r.pool.Begin(ctx)
|
||
if err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
j, err := lockJob(ctx, tx, jobID)
|
||
if errors.Is(err, repository.ErrNotFound) {
|
||
return Terminal, nil, nil, nil
|
||
}
|
||
if err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
var now time.Time
|
||
if err := tx.QueryRow(ctx, `SELECT now()`).Scan(&now); err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
switch {
|
||
case j.State == "done":
|
||
return AlreadyDone, j, nil, nil
|
||
case j.State == "failed":
|
||
return Terminal, j, nil, nil
|
||
case j.State == "processing" && j.LeaseUntil != nil && j.LeaseUntil.After(now):
|
||
return Busy, j, nil, nil
|
||
case j.State == "queued" && j.NextAttemptAt.After(now): // время relay и claim — одно (часы PostgreSQL)
|
||
return NotDue, j, nil, nil
|
||
}
|
||
a, err := scanAttachment(tx.QueryRow(ctx, `SELECT `+attachmentColumns+` FROM attachments WHERE id = $1::uuid FOR UPDATE`, j.AttachmentID))
|
||
if errors.Is(err, repository.ErrNotFound) {
|
||
return Terminal, j, nil, nil
|
||
}
|
||
if err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
// Результат этой или более новой версии уже опубликован — повтор не нужен.
|
||
if a.Status == domain.AttachmentReady && a.PipelineVersion >= j.PipelineVersion {
|
||
if _, err := tx.Exec(ctx, `UPDATE jobs SET state = 'done', lease_until = NULL, updated_at = now() WHERE job_id = $1::uuid`, jobID); err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
if err := event(ctx, tx, jobID, "noop_already_published", fmt.Sprintf("published_version=%d", a.PipelineVersion)); err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
return AlreadyDone, j, a, tx.Commit(ctx)
|
||
}
|
||
takeover := j.State == "processing"
|
||
j.Attempt++
|
||
if j.Attempt > j.MaxAttempts {
|
||
if err := failTx(ctx, tx, j, "retry_exhausted", "attempts exhausted at claim"); err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
return ExhaustedNow, j, a, tx.Commit(ctx)
|
||
}
|
||
until := now.Add(lease)
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE jobs SET state = 'processing', attempt = $2, lease_until = $3, lease_owner = $4, updated_at = now()
|
||
WHERE job_id = $1::uuid`, jobID, j.Attempt, until, owner); err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
if a.Status == domain.AttachmentQueued {
|
||
if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = $1::uuid`, a.ID); err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
}
|
||
ev := "claimed"
|
||
if takeover {
|
||
ev = "claimed_after_lease_expiry"
|
||
}
|
||
if err := event(ctx, tx, jobID, ev, fmt.Sprintf("attempt=%d owner=%s", j.Attempt, owner)); err != nil {
|
||
return 0, nil, nil, err
|
||
}
|
||
j.State, j.LeaseUntil, j.LeaseOwner = "processing", &until, owner
|
||
return Claimed, j, a, tx.Commit(ctx)
|
||
}
|
||
|
||
// ExtendLease продлевает аренду (heartbeat), пока worker действительно работает.
|
||
// false — аренда потеряна (её забрал другой worker после истечения).
|
||
func (r *PipelineRepository) ExtendLease(ctx context.Context, jobID, owner string, attempt int, lease time.Duration) (bool, error) {
|
||
tag, err := r.pool.Exec(ctx, `
|
||
UPDATE jobs SET lease_until = now() + make_interval(secs => $4), updated_at = now()
|
||
WHERE job_id = $1::uuid AND state = 'processing' AND lease_owner = $2 AND attempt = $3`,
|
||
jobID, owner, attempt, lease.Seconds())
|
||
return tag.RowsAffected() == 1, err
|
||
}
|
||
|
||
// CompleteResult — что сделала фиксация.
|
||
type CompleteResult struct {
|
||
Applied bool // результат этой попытки зафиксирован
|
||
Published bool // attachment переведён на эту версию
|
||
LostLease bool // fencing: попытка уже не владеет job
|
||
NewerPublished int // если опубликована более новая версия — её номер
|
||
}
|
||
|
||
// Complete атомарно публикует результат: варианты (ON CONFLICT DO NOTHING по
|
||
// ключу эффекта), attachment → ready на этой версии (только если опубликованная
|
||
// версия меньше — старый результат не перезаписывает новый), job → done.
|
||
func (r *PipelineRepository) Complete(ctx context.Context, jobID, owner string, attempt int, finalKey string, variants []domain.Variant) (CompleteResult, error) {
|
||
var res CompleteResult
|
||
tx, err := r.pool.Begin(ctx)
|
||
if err != nil {
|
||
return res, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
j, err := lockJob(ctx, tx, jobID)
|
||
if err != nil {
|
||
return res, err
|
||
}
|
||
if j.State != "processing" || j.LeaseOwner != owner || j.Attempt != attempt {
|
||
res.LostLease = true
|
||
return res, event(ctx, tx, jobID, "result_rejected_lost_lease", fmt.Sprintf("attempt=%d owner=%s", attempt, owner))
|
||
}
|
||
batch := &pgx.Batch{}
|
||
for _, v := range variants {
|
||
batch.Queue(`
|
||
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256)
|
||
VALUES ($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9)
|
||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING`,
|
||
j.AttachmentID, j.PipelineVersion, v.Name, v.ObjectKey, v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256)
|
||
}
|
||
if err := tx.SendBatch(ctx, batch).Close(); err != nil {
|
||
return res, fmt.Errorf("insert variants: %w", err)
|
||
}
|
||
tag, err := tx.Exec(ctx, `
|
||
UPDATE attachments
|
||
SET status = 'ready', pipeline_version = $2, object_key = $3, error_code = NULL, updated_at = now()
|
||
WHERE id = $1::uuid AND pipeline_version < $2 AND status <> 'missing'`,
|
||
j.AttachmentID, j.PipelineVersion, finalKey)
|
||
if err != nil {
|
||
return res, fmt.Errorf("publish attachment: %w", err)
|
||
}
|
||
res.Published = tag.RowsAffected() == 1
|
||
if !res.Published {
|
||
_ = tx.QueryRow(ctx, `SELECT pipeline_version FROM attachments WHERE id = $1::uuid`, j.AttachmentID).Scan(&res.NewerPublished)
|
||
}
|
||
if _, err := tx.Exec(ctx, `UPDATE jobs SET state = 'done', lease_until = NULL, last_error = NULL, updated_at = now() WHERE job_id = $1::uuid`, jobID); err != nil {
|
||
return res, err
|
||
}
|
||
ev, detail := "done", fmt.Sprintf("attempt=%d", attempt)
|
||
if !res.Published {
|
||
ev, detail = "done_not_published_newer_exists", fmt.Sprintf("attempt=%d published_version=%d", attempt, res.NewerPublished)
|
||
}
|
||
if err := event(ctx, tx, jobID, ev, detail); err != nil {
|
||
return res, err
|
||
}
|
||
res.Applied = true
|
||
return res, tx.Commit(ctx)
|
||
}
|
||
|
||
func failTx(ctx context.Context, tx pgx.Tx, j *JobRow, code, detail string) error {
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE jobs SET state = 'failed', lease_until = NULL, last_error = $2, updated_at = now()
|
||
WHERE job_id = $1::uuid`, j.JobID, code+": "+detail); err != nil {
|
||
return err
|
||
}
|
||
// Ready-вложение (переобработка новой версии) не роняется в failed: старый
|
||
// результат остаётся опубликованным.
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now()
|
||
WHERE id = $1::uuid AND status IN ('queued', 'processing')`, j.AttachmentID, code); err != nil {
|
||
return err
|
||
}
|
||
return event(ctx, tx, j.JobID, "failed", code+": "+detail)
|
||
}
|
||
|
||
// RetryOrFail: временная ошибка с оставшимися попытками → queued с backoff и
|
||
// новым сообщением outbox (available_at в будущем); иначе → failed.
|
||
func (r *PipelineRepository) RetryOrFail(ctx context.Context, jobID, owner string, attempt int, code, detail string, permanent bool, backoff time.Duration) (failed bool, err error) {
|
||
tx, err := r.pool.Begin(ctx)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
j, err := lockJob(ctx, tx, jobID)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if j.State != "processing" || j.LeaseOwner != owner || j.Attempt != attempt {
|
||
return false, event(ctx, tx, jobID, "error_ignored_lost_lease", code)
|
||
}
|
||
if permanent || j.Attempt >= j.MaxAttempts {
|
||
if !permanent {
|
||
code = "retry_exhausted"
|
||
}
|
||
if err := failTx(ctx, tx, j, code, detail); err != nil {
|
||
return false, err
|
||
}
|
||
return true, tx.Commit(ctx)
|
||
}
|
||
var next time.Time
|
||
if err := tx.QueryRow(ctx, `
|
||
UPDATE jobs SET state = 'queued', lease_until = NULL, lease_owner = NULL, last_error = $2,
|
||
next_attempt_at = now() + make_interval(secs => $3), updated_at = now()
|
||
WHERE job_id = $1::uuid RETURNING next_attempt_at`, jobID, code+": "+detail, backoff.Seconds()).Scan(&next); err != nil {
|
||
return false, err
|
||
}
|
||
if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1::uuid AND status = 'processing'`, j.AttachmentID); err != nil {
|
||
return false, err
|
||
}
|
||
if err := outboxTx(ctx, tx, jobID, j.AttachmentID, j.PipelineVersion, j.Attempt+1, next); err != nil {
|
||
return false, err
|
||
}
|
||
if err := event(ctx, tx, jobID, "retry_scheduled", fmt.Sprintf("%s; next_attempt=%d in %s", code, j.Attempt+1, backoff.Round(time.Millisecond))); err != nil {
|
||
return false, err
|
||
}
|
||
return false, tx.Commit(ctx)
|
||
}
|
||
|
||
// SweepExpiredLeases возвращает в очередь работу, чей worker умер (lease истёк).
|
||
func (r *PipelineRepository) SweepExpiredLeases(ctx context.Context) (int, error) {
|
||
tx, err := r.pool.Begin(ctx)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
rows, err := tx.Query(ctx, `
|
||
SELECT job_id::text FROM jobs
|
||
WHERE state = 'processing' AND lease_until < now()
|
||
ORDER BY lease_until LIMIT 100 FOR UPDATE SKIP LOCKED`)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
var ids []string
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
rows.Close()
|
||
return 0, err
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
rows.Close()
|
||
for _, id := range ids {
|
||
j, err := lockJob(ctx, tx, id)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
if j.Attempt >= j.MaxAttempts {
|
||
if err := failTx(ctx, tx, j, "retry_exhausted", "lease expired on last attempt"); err != nil {
|
||
return 0, err
|
||
}
|
||
continue
|
||
}
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE jobs SET state = 'queued', lease_until = NULL, lease_owner = NULL, next_attempt_at = now(),
|
||
last_error = 'lease_expired', updated_at = now() WHERE job_id = $1::uuid`, id); err != nil {
|
||
return 0, err
|
||
}
|
||
if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1::uuid AND status = 'processing'`, j.AttachmentID); err != nil {
|
||
return 0, err
|
||
}
|
||
if err := outboxTx(ctx, tx, id, j.AttachmentID, j.PipelineVersion, j.Attempt+1, time.Now()); err != nil {
|
||
return 0, err
|
||
}
|
||
if err := event(ctx, tx, id, "lease_expired_requeued", fmt.Sprintf("owner=%s attempt=%d", j.LeaseOwner, j.Attempt)); err != nil {
|
||
return 0, err
|
||
}
|
||
}
|
||
return len(ids), tx.Commit(ctx)
|
||
}
|
||
|
||
// OutboxMessage — неопубликованное сообщение.
|
||
type OutboxMessage struct {
|
||
ID int64
|
||
JobID string
|
||
Payload []byte
|
||
Trace map[string]string // W3C trace context родителя (upload / попытка)
|
||
Created time.Time
|
||
}
|
||
|
||
// PublishDue публикует созревшие сообщения outbox. Строки блокируются
|
||
// FOR UPDATE SKIP LOCKED (несколько relay не мешают друг другу); published_at
|
||
// ставится только после publisher confirm. Падение между confirm и COMMIT даёт
|
||
// повторную публикацию — это допустимо: consumer идемпотентен.
|
||
func (r *PipelineRepository) PublishDue(ctx context.Context, limit int, publish func(context.Context, OutboxMessage) error, afterPublish func()) (int, error) {
|
||
tx, err := r.pool.Begin(ctx)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
rows, err := tx.Query(ctx, `
|
||
SELECT id, job_id::text, payload, trace_context, created_at FROM outbox
|
||
WHERE published_at IS NULL AND available_at <= now()
|
||
ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED`, limit)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
var msgs []OutboxMessage
|
||
for rows.Next() {
|
||
var m OutboxMessage
|
||
if err := rows.Scan(&m.ID, &m.JobID, &m.Payload, &m.Trace, &m.Created); err != nil {
|
||
rows.Close()
|
||
return 0, err
|
||
}
|
||
msgs = append(msgs, m)
|
||
}
|
||
rows.Close()
|
||
sent := 0
|
||
for _, m := range msgs {
|
||
if err := publish(ctx, m); err != nil {
|
||
_, _ = tx.Exec(ctx, `UPDATE outbox SET publish_attempts = publish_attempts + 1 WHERE id = $1`, m.ID)
|
||
_ = tx.Commit(ctx)
|
||
return sent, err
|
||
}
|
||
if afterPublish != nil {
|
||
afterPublish() // точка внедрения сбоя «после confirm до COMMIT»
|
||
}
|
||
if _, err := tx.Exec(ctx, `UPDATE outbox SET published_at = now(), publish_attempts = publish_attempts + 1 WHERE id = $1`, m.ID); err != nil {
|
||
return sent, err
|
||
}
|
||
sent++
|
||
}
|
||
return sent, tx.Commit(ctx)
|
||
}
|
||
|
||
// Replay — явный операторский повтор failed-работы (failed → queued) с аудитом.
|
||
// Успешную работу повторить нельзя: «повторить» не означает исполнить дубль.
|
||
func (r *PipelineRepository) Replay(ctx context.Context, attachmentID string, version int, actor string) (string, error) {
|
||
tx, err := r.pool.Begin(ctx)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
var jobID string
|
||
if err := tx.QueryRow(ctx, `SELECT job_id::text FROM jobs WHERE attachment_id = $1::uuid AND pipeline_version = $2`,
|
||
attachmentID, version).Scan(&jobID); err != nil {
|
||
if errors.Is(err, pgx.ErrNoRows) {
|
||
return "", repository.ErrNotFound
|
||
}
|
||
return "", err
|
||
}
|
||
j, err := lockJob(ctx, tx, jobID)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if j.State != "failed" {
|
||
_ = event(ctx, tx, jobID, "replay_refused", "state="+j.State+" actor="+actor)
|
||
_ = tx.Commit(ctx)
|
||
return jobID, fmt.Errorf("%w: job is %s, only failed jobs can be replayed", repository.ErrConflict, j.State)
|
||
}
|
||
var staging *string
|
||
if err := tx.QueryRow(ctx, `SELECT staging_key FROM attachments WHERE id = $1::uuid FOR UPDATE`, attachmentID).Scan(&staging); err != nil {
|
||
return "", err
|
||
}
|
||
if staging == nil {
|
||
_ = event(ctx, tx, jobID, "replay_refused", "staging removed by retention, actor="+actor)
|
||
_ = tx.Commit(ctx)
|
||
return jobID, fmt.Errorf("%w: source removed by staging retention", repository.ErrConflict)
|
||
}
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE jobs SET state = 'queued', attempt = 0, next_attempt_at = now(), last_error = NULL, updated_at = now()
|
||
WHERE job_id = $1::uuid`, jobID); err != nil {
|
||
return "", err
|
||
}
|
||
if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = $1::uuid AND status = 'failed'`, attachmentID); err != nil {
|
||
return "", err
|
||
}
|
||
if err := outboxTx(ctx, tx, jobID, attachmentID, version, 1, time.Now()); err != nil {
|
||
return "", err
|
||
}
|
||
if err := event(ctx, tx, jobID, "replay", "actor="+actor); err != nil {
|
||
return "", err
|
||
}
|
||
return jobID, tx.Commit(ctx)
|
||
}
|
||
|
||
// Reprocess ставит работу новой версии pipeline для уже обработанного вложения
|
||
// (трек 2). Вложение остаётся ready со старыми вариантами до публикации новых.
|
||
func (r *PipelineRepository) Reprocess(ctx context.Context, attachmentID string, version, maxAttempts int) (string, error) {
|
||
tx, err := r.pool.Begin(ctx)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
id, err := EnqueueJobTx(ctx, tx, attachmentID, version, maxAttempts, "reprocess")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return id, tx.Commit(ctx)
|
||
}
|
||
|
||
// FailedJobs — реестр failed для оператора.
|
||
func (r *PipelineRepository) FailedJobs(ctx context.Context, limit int) ([]JobRow, error) {
|
||
rows, err := r.pool.Query(ctx, `
|
||
SELECT job_id::text, attachment_id::text, pipeline_version, state, attempt, max_attempts, COALESCE(last_error, '')
|
||
FROM jobs WHERE state = 'failed' ORDER BY updated_at DESC LIMIT $1`, limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []JobRow
|
||
for rows.Next() {
|
||
var j JobRow
|
||
if err := rows.Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.State, &j.Attempt, &j.MaxAttempts, &j.LastError); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, j)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// StagingCandidate — staging-объект, который можно удалить.
|
||
type StagingCandidate struct {
|
||
AttachmentID string
|
||
Key string
|
||
Status string
|
||
}
|
||
|
||
// StagingToClean: ready с отдельным staging-объектом (результат уже опубликован
|
||
// не менее readyGrace назад) и failed старше retention (replay больше невозможен).
|
||
func (r *PipelineRepository) StagingToClean(ctx context.Context, readyGrace, failedRetention time.Duration, limit int) ([]StagingCandidate, error) {
|
||
rows, err := r.pool.Query(ctx, `
|
||
SELECT id::text, staging_key, status FROM attachments
|
||
WHERE staging_key IS NOT NULL AND (
|
||
(status = 'ready' AND staging_key <> object_key AND updated_at < now() - make_interval(secs => $1))
|
||
OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
|
||
ORDER BY updated_at LIMIT $3`, readyGrace.Seconds(), failedRetention.Seconds(), limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []StagingCandidate
|
||
for rows.Next() {
|
||
var c StagingCandidate
|
||
if err := rows.Scan(&c.AttachmentID, &c.Key, &c.Status); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, c)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// ClearStaging отмечает, что staging-объект удалён (только если ключ не менялся).
|
||
func (r *PipelineRepository) ClearStaging(ctx context.Context, attachmentID, key string) error {
|
||
_, err := r.pool.Exec(ctx, `UPDATE attachments SET staging_key = NULL WHERE id = $1::uuid AND staging_key = $2`, attachmentID, key)
|
||
return err
|
||
}
|
||
|
||
// Attachment — доверенные метаданные для worker.
|
||
func (r *PipelineRepository) Attachment(ctx context.Context, id string) (*domain.Attachment, error) {
|
||
return scanAttachment(r.pool.QueryRow(ctx, `SELECT `+attachmentColumns+` FROM attachments WHERE id = $1::uuid`, id))
|
||
}
|
||
|
||
// traceJSON — W3C trace context активного span для колонки trace_context (nil, если span нет).
|
||
func traceJSON(ctx context.Context) []byte {
|
||
m := telemetry.InjectMap(ctx)
|
||
if m == nil {
|
||
return nil
|
||
}
|
||
b, _ := json.Marshal(m)
|
||
return b
|
||
}
|
||
|
||
// QueueStats — состояние очереди по источнику истины (PostgreSQL), для метрик:
|
||
// число работ по состояниям и возраст старейшей незавершённой работы/сообщения.
|
||
type QueueStats struct {
|
||
Queued, Processing, Failed int64
|
||
OldestQueuedAge float64 // s, 0 если очередь пуста
|
||
OutboxPending int64
|
||
OldestOutboxAge float64 // s
|
||
}
|
||
|
||
func (r *PipelineRepository) QueueStats(ctx context.Context) (QueueStats, error) {
|
||
var s QueueStats
|
||
err := r.pool.QueryRow(ctx, `
|
||
SELECT count(*) FILTER (WHERE state = 'queued'),
|
||
count(*) FILTER (WHERE state = 'processing'),
|
||
count(*) FILTER (WHERE state = 'failed'),
|
||
COALESCE(EXTRACT(EPOCH FROM now() - min(created_at) FILTER (WHERE state IN ('queued', 'processing'))), 0)
|
||
FROM jobs`).Scan(&s.Queued, &s.Processing, &s.Failed, &s.OldestQueuedAge)
|
||
if err != nil {
|
||
return s, err
|
||
}
|
||
err = r.pool.QueryRow(ctx, `
|
||
SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now() - min(created_at)), 0)
|
||
FROM outbox WHERE published_at IS NULL AND available_at <= now()`).Scan(&s.OutboxPending, &s.OldestOutboxAge)
|
||
return s, err
|
||
}
|