forked from mtusi2026/mtusi2026lab6
fix: persist attachment SLO outcomes
This commit is contained in:
@@ -17,7 +17,6 @@ type Metrics struct {
|
||||
workerInFlight metric.Int64UpDownCounter
|
||||
workerJobs metric.Int64Counter
|
||||
workerDuration metric.Float64Histogram
|
||||
workerTimeToReady metric.Float64Histogram
|
||||
workerRetries metric.Int64Counter
|
||||
workerReclaimed metric.Int64Counter
|
||||
outboxPublished metric.Int64Counter
|
||||
@@ -39,10 +38,6 @@ func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *M
|
||||
metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
|
||||
)
|
||||
workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready",
|
||||
metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(workerDurationBuckets...),
|
||||
)
|
||||
workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt."))
|
||||
workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout."))
|
||||
outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams."))
|
||||
@@ -59,8 +54,8 @@ func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *M
|
||||
}
|
||||
return &Metrics{
|
||||
meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs,
|
||||
workerDuration: workerDuration, workerTimeToReady: workerTimeToReady,
|
||||
workerRetries: workerRetries, workerReclaimed: workerReclaimed,
|
||||
workerDuration: workerDuration,
|
||||
workerRetries: workerRetries, workerReclaimed: workerReclaimed,
|
||||
outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure,
|
||||
outboxRecovered: outboxRecovered, handler: handler,
|
||||
}
|
||||
@@ -81,10 +76,6 @@ func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration t
|
||||
m.workerDuration.Record(ctx, duration.Seconds(), attributes)
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) {
|
||||
m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process")))
|
||||
}
|
||||
|
||||
func (m *Metrics) ObserveRetry(ctx context.Context) {
|
||||
m.workerRetries.Add(ctx, 1)
|
||||
}
|
||||
|
||||
@@ -58,7 +58,6 @@ type ImageProcessor interface {
|
||||
type Metrics interface {
|
||||
ObserveStarted(context.Context)
|
||||
ObserveFinished(context.Context, string, time.Duration)
|
||||
ObserveTimeToReady(context.Context, time.Duration)
|
||||
ObserveRetry(context.Context)
|
||||
ObserveReclaimed(context.Context, int)
|
||||
}
|
||||
@@ -290,7 +289,6 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv
|
||||
return err
|
||||
}
|
||||
result = "ready"
|
||||
w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -350,7 +348,7 @@ func (w *Worker) resolveClaim(
|
||||
if job.ErrorCode != nil {
|
||||
code = *job.ErrorCode
|
||||
}
|
||||
return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
return "duplicate_failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()})
|
||||
default:
|
||||
return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr)
|
||||
}
|
||||
@@ -538,6 +536,5 @@ type nopMetrics struct{}
|
||||
|
||||
func (nopMetrics) ObserveStarted(context.Context) {}
|
||||
func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {}
|
||||
func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {}
|
||||
func (nopMetrics) ObserveRetry(context.Context) {}
|
||||
func (nopMetrics) ObserveReclaimed(context.Context, int) {}
|
||||
|
||||
@@ -37,9 +37,6 @@ func (s *Store) QueueAttachmentWithJobAndOutbox(
|
||||
return model.Attachment{}, model.ErrInvalidInput
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if job.CreatedAt.IsZero() {
|
||||
job.CreatedAt = now
|
||||
}
|
||||
if job.AvailableAt.IsZero() {
|
||||
job.AvailableAt = now
|
||||
}
|
||||
@@ -67,8 +64,7 @@ func (s *Store) QueueAttachmentWithJobAndOutbox(
|
||||
|
||||
job.AttachmentID = attachmentID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = job.CreatedAt
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
if err := insertAttachmentJob(ctx, tx, &job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
@@ -92,7 +88,6 @@ func (s *Store) QueueAttachmentPipelineVersion(
|
||||
}
|
||||
job.Attempt = 1
|
||||
now := time.Now().UTC()
|
||||
job.CreatedAt = now
|
||||
job.AvailableAt = now
|
||||
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
@@ -114,7 +109,6 @@ func (s *Store) QueueAttachmentPipelineVersion(
|
||||
}
|
||||
job.AttachmentID = attachment.ID
|
||||
job.Status = model.AttachmentJobQueued
|
||||
job.UpdatedAt = now
|
||||
const queueAttachment = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', pipeline_version = $2, staging_key = object_key,
|
||||
@@ -124,7 +118,7 @@ func (s *Store) QueueAttachmentPipelineVersion(
|
||||
if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil {
|
||||
return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err)
|
||||
}
|
||||
if err := insertAttachmentJob(ctx, tx, job); err != nil {
|
||||
if err := insertAttachmentJob(ctx, tx, &job); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil {
|
||||
@@ -163,15 +157,16 @@ func (s *Store) alreadyQueuedAttachment(
|
||||
return publicAttachment(attachment), nil
|
||||
}
|
||||
|
||||
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error {
|
||||
func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job *model.AttachmentJob) error {
|
||||
const query = `
|
||||
INSERT INTO attachment_jobs (
|
||||
id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)`
|
||||
if _, err := tx.Exec(
|
||||
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt,
|
||||
); err != nil {
|
||||
VALUES ($1, $2, $3, $4, 'queued', $5, statement_timestamp(), statement_timestamp())
|
||||
RETURNING created_at, updated_at`
|
||||
if err := tx.QueryRow(
|
||||
ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt,
|
||||
).Scan(&job.CreatedAt, &job.UpdatedAt); err != nil {
|
||||
return fmt.Errorf("insert attachment job: %w", err)
|
||||
}
|
||||
return nil
|
||||
@@ -417,6 +412,9 @@ func (s *Store) CompleteAttachmentJob(
|
||||
if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil {
|
||||
return "", fmt.Errorf("complete attachment: %w", err)
|
||||
}
|
||||
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobReady); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit completed attachment job: %w", err)
|
||||
}
|
||||
@@ -527,12 +525,26 @@ func (s *Store) FailAttachmentJob(
|
||||
if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil {
|
||||
return "", fmt.Errorf("fail attachment: %w", err)
|
||||
}
|
||||
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", fmt.Errorf("commit failed attachment job: %w", err)
|
||||
}
|
||||
return model.JobMutationApplied, nil
|
||||
}
|
||||
|
||||
func insertAttachmentTerminalEvent(ctx context.Context, tx pgx.Tx, job model.AttachmentJob, status string) error {
|
||||
const query = `
|
||||
INSERT INTO attachment_terminal_events (job_id, status, accepted_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (job_id) DO NOTHING`
|
||||
if _, err := tx.Exec(ctx, query, job.ID, status, job.CreatedAt); err != nil {
|
||||
return fmt.Errorf("insert attachment terminal event: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockJobMutation(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
|
||||
@@ -89,9 +89,7 @@ func (s *Store) RecoverExpiredAttachmentJobs(
|
||||
if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil {
|
||||
return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil {
|
||||
return model.RecoveryResult{}, err
|
||||
}
|
||||
recovered.Requeued++
|
||||
@@ -118,6 +116,9 @@ func markExpiredJobFailed(
|
||||
if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil {
|
||||
return fmt.Errorf("fail expired attachment job: %w", err)
|
||||
}
|
||||
if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil {
|
||||
return err
|
||||
}
|
||||
if !failAttachment {
|
||||
return nil
|
||||
}
|
||||
@@ -198,9 +199,7 @@ func (s *Store) ReplayFailedAttachment(
|
||||
); err != nil {
|
||||
return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err)
|
||||
}
|
||||
eventJob := job
|
||||
eventJob.CreatedAt = time.Now().UTC()
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil {
|
||||
if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil {
|
||||
return model.AttachmentJob{}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab6/migrations"
|
||||
)
|
||||
|
||||
type PostgresSuite struct {
|
||||
@@ -210,6 +212,7 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
s.assertTerminalEvent(job.ID, model.AttachmentJobReady, lease.Job.CreatedAt)
|
||||
result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationAlreadyComplete, result)
|
||||
@@ -252,6 +255,130 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() {
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentJobCreationUsesDatabaseClock() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachmentID := uuid.NewString()
|
||||
stagingKey := "staging/" + attachmentID + "/original"
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: attachmentID, ChatID: chatID, OwnerID: owner.ID, Filename: "future.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("f", 64), ObjectKey: stagingKey,
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
future := time.Now().Add(24 * time.Hour)
|
||||
job := model.AttachmentJob{
|
||||
ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1,
|
||||
Attempt: 1, CreatedAt: future,
|
||||
}
|
||||
_, err = s.store.QueueAttachmentWithJobAndOutbox(context.Background(), pending.ID, stagingKey, job, "", "")
|
||||
s.Require().NoError(err)
|
||||
stored, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
s.True(stored.CreatedAt.Before(future))
|
||||
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachmentID, 1, 1, "worker-clock", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachmentID, 1))
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
s.assertTerminalEvent(job.ID, model.AttachmentJobReady, stored.CreatedAt)
|
||||
s.assertOutboxCreationTime(job.ID, 1, stored.CreatedAt)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestTerminalLedgerMigrationBackfillsExistingJob() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
replayedAttachment, replayedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
skewedAttachment, skewedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
replayedAcceptedAt := time.Now().Add(-2 * time.Minute).UTC().Truncate(time.Microsecond)
|
||||
firstFailureAt := replayedAcceptedAt.Add(20 * time.Second)
|
||||
replayedReadyAt := firstFailureAt.Add(time.Minute)
|
||||
skewedTerminalAt := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond)
|
||||
skewedCreatedAt := skewedTerminalAt.Add(time.Minute)
|
||||
tx, err := s.pool.Begin(context.Background())
|
||||
s.Require().NoError(err)
|
||||
defer func() { s.Require().NoError(tx.Rollback(context.Background())) }()
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET status = 'ready', attempt = 2, error_code = NULL, created_at = $2, updated_at = $3
|
||||
WHERE id = $1`,
|
||||
replayedJob.ID,
|
||||
replayedAcceptedAt,
|
||||
replayedReadyAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = $2
|
||||
WHERE id = $1`,
|
||||
replayedAttachment.ID,
|
||||
replayedReadyAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`INSERT INTO attachment_replay_audit (
|
||||
id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason, created_at
|
||||
) VALUES ($1, $2, $3, 1, 2, 'legacy-operator', 'legacy replay', $4)`,
|
||||
uuid.NewString(),
|
||||
replayedJob.ID,
|
||||
replayedAttachment.ID,
|
||||
firstFailureAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET status = 'failed', error_code = 'legacy_failure', created_at = $2, updated_at = $3
|
||||
WHERE id = $1`,
|
||||
skewedJob.ID,
|
||||
skewedCreatedAt,
|
||||
skewedTerminalAt,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachments SET status = 'failed', error_code = 'legacy_failure', ready_at = NULL
|
||||
WHERE id = $1`,
|
||||
skewedAttachment.ID,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(context.Background(), `ALTER INDEX attachment_terminal_events_status_terminal_at_idx RENAME TO attachment_terminal_events_status_terminal_at_idx_old`)
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(context.Background(), `ALTER TABLE attachment_terminal_events RENAME TO attachment_terminal_events_old`)
|
||||
s.Require().NoError(err)
|
||||
migration, err := migrations.FS.ReadFile("0008_attachment_terminal_events.sql")
|
||||
s.Require().NoError(err)
|
||||
_, err = tx.Exec(context.Background(), string(migration), pgx.QueryExecModeSimpleProtocol)
|
||||
s.Require().NoError(err)
|
||||
|
||||
s.assertTerminalEventInTx(tx, replayedJob.ID, model.AttachmentJobFailed, replayedAcceptedAt, firstFailureAt)
|
||||
s.assertTerminalEventInTx(tx, skewedJob.ID, model.AttachmentJobFailed, skewedTerminalAt, skewedTerminalAt)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) assertTerminalEventInTx(
|
||||
tx pgx.Tx,
|
||||
jobID string,
|
||||
status string,
|
||||
acceptedAt time.Time,
|
||||
terminalAt time.Time,
|
||||
) {
|
||||
s.T().Helper()
|
||||
var (
|
||||
actualStatus string
|
||||
actualAcceptedAt time.Time
|
||||
actualTerminalAt time.Time
|
||||
)
|
||||
err := tx.QueryRow(
|
||||
context.Background(),
|
||||
`SELECT status, accepted_at, terminal_at FROM attachment_terminal_events WHERE job_id = $1`,
|
||||
jobID,
|
||||
).Scan(&actualStatus, &actualAcceptedAt, &actualTerminalAt)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(status, actualStatus)
|
||||
s.True(acceptedAt.Equal(actualAcceptedAt))
|
||||
s.True(terminalAt.Equal(actualTerminalAt))
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
_, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
@@ -295,6 +422,8 @@ func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() {
|
||||
func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
original, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
lease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
@@ -302,6 +431,7 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image")
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)
|
||||
|
||||
_, _, err = s.store.CreateMessage(
|
||||
context.Background(), chatID, owner.ID, "failed attachment",
|
||||
@@ -320,6 +450,18 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() {
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, replayed.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, replayed.Status)
|
||||
s.Equal(original.CreatedAt, replayed.CreatedAt)
|
||||
s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt)
|
||||
replayedLease, err := s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, replayed.PipelineVersion, replayed.Attempt, "worker-2", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
result, err = s.store.CompleteAttachmentJob(
|
||||
context.Background(), replayedLease, completedVariants(attachment.ID, replayed.PipelineVersion),
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationApplied, result)
|
||||
s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt))
|
||||
|
||||
var auditCount int
|
||||
err = s.pool.QueryRow(
|
||||
@@ -336,6 +478,8 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
original, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.pool.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
|
||||
@@ -351,12 +495,82 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() {
|
||||
s.Require().NoError(err)
|
||||
s.Equal(2, current.Attempt)
|
||||
s.Equal(model.AttachmentJobQueued, current.Status)
|
||||
s.Equal(original.CreatedAt, current.CreatedAt)
|
||||
s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt)
|
||||
|
||||
result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1))
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.JobMutationStale, result)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestExpiredLeaseRecoveryFailsExhaustedJobOnce() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
original, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.store.ClaimAttachmentJob(
|
||||
context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
_, err = s.pool.Exec(
|
||||
context.Background(),
|
||||
`UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`,
|
||||
job.ID,
|
||||
)
|
||||
s.Require().NoError(err)
|
||||
|
||||
recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now())
|
||||
s.Require().NoError(err)
|
||||
s.Zero(recovered.Requeued)
|
||||
s.Equal(1, recovered.Failed)
|
||||
firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)
|
||||
|
||||
recovered, err = s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now())
|
||||
s.Require().NoError(err)
|
||||
s.Zero(recovered.Requeued)
|
||||
s.Zero(recovered.Failed)
|
||||
current, err := s.store.AttachmentJob(context.Background(), job.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentJobFailed, current.Status)
|
||||
s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt))
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) assertTerminalEvent(jobID string, status string, acceptedAt time.Time) time.Time {
|
||||
s.T().Helper()
|
||||
var (
|
||||
actualStatus string
|
||||
actualAcceptedAt time.Time
|
||||
terminalAt time.Time
|
||||
count int
|
||||
)
|
||||
err := s.pool.QueryRow(
|
||||
context.Background(),
|
||||
`SELECT min(status), min(accepted_at), min(terminal_at), count(*)
|
||||
FROM attachment_terminal_events WHERE job_id = $1`,
|
||||
jobID,
|
||||
).Scan(&actualStatus, &actualAcceptedAt, &terminalAt, &count)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(1, count)
|
||||
s.Equal(status, actualStatus)
|
||||
s.True(acceptedAt.Equal(actualAcceptedAt), "expected %s, got %s", acceptedAt, actualAcceptedAt)
|
||||
return terminalAt
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) assertOutboxCreationTime(jobID string, attempt int, expected time.Time) {
|
||||
s.T().Helper()
|
||||
var payload []byte
|
||||
err := s.pool.QueryRow(
|
||||
context.Background(),
|
||||
`SELECT payload FROM outbox_events WHERE job_id = $1 AND attempt = $2`,
|
||||
jobID,
|
||||
attempt,
|
||||
).Scan(&payload)
|
||||
s.Require().NoError(err)
|
||||
var message model.AttachmentJobMessage
|
||||
s.Require().NoError(json.Unmarshal(payload, &message))
|
||||
s.True(expected.Equal(message.CreatedAt), "expected %s, got %s", expected, message.CreatedAt)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
CREATE TABLE attachment_terminal_events (
|
||||
job_id UUID PRIMARY KEY,
|
||||
status TEXT NOT NULL,
|
||||
accepted_at TIMESTAMPTZ NOT NULL,
|
||||
terminal_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachment_terminal_events_status CHECK (status IN ('ready', 'failed')),
|
||||
CONSTRAINT attachment_terminal_events_time CHECK (terminal_at >= accepted_at)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_terminal_events_status_terminal_at_idx
|
||||
ON attachment_terminal_events (status, terminal_at);
|
||||
|
||||
WITH first_replay AS (
|
||||
SELECT job_id, MIN(created_at) AS created_at
|
||||
FROM attachment_replay_audit
|
||||
GROUP BY job_id
|
||||
)
|
||||
INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at)
|
||||
SELECT jobs.id, 'failed', LEAST(jobs.created_at, first_replay.created_at), first_replay.created_at
|
||||
FROM attachment_jobs jobs
|
||||
JOIN first_replay ON first_replay.job_id = jobs.id
|
||||
ON CONFLICT (job_id) DO NOTHING;
|
||||
|
||||
INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at)
|
||||
SELECT jobs.id, jobs.status, LEAST(jobs.created_at, jobs.updated_at), jobs.updated_at
|
||||
FROM attachment_jobs jobs
|
||||
WHERE jobs.status IN ('ready', 'failed')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM attachment_replay_audit replay
|
||||
WHERE replay.job_id = jobs.id
|
||||
)
|
||||
ON CONFLICT (job_id) DO NOTHING;
|
||||
@@ -37,3 +37,61 @@ messenger_outbox:
|
||||
- oldest_pending_age_seconds:
|
||||
usage: GAUGE
|
||||
description: Age of the oldest unpublished outbox event.
|
||||
|
||||
messenger_attachment_terminal_transitions:
|
||||
query: |
|
||||
WITH statuses(status) AS (
|
||||
VALUES ('ready'), ('failed')
|
||||
)
|
||||
SELECT
|
||||
statuses.status,
|
||||
COUNT(events.job_id)::bigint AS total
|
||||
FROM statuses
|
||||
LEFT JOIN attachment_terminal_events events ON events.status = statuses.status
|
||||
GROUP BY statuses.status
|
||||
metrics:
|
||||
- status:
|
||||
usage: LABEL
|
||||
description: First terminal status recorded for a durable attachment job.
|
||||
- total:
|
||||
usage: COUNTER
|
||||
description: Durable attachment jobs whose first terminal outcome has this status.
|
||||
|
||||
messenger_attachment_time_to_ready_seconds:
|
||||
query: |
|
||||
WITH bounds(le, upper_bound) AS (
|
||||
VALUES
|
||||
('0.05', 0.05::double precision),
|
||||
('0.1', 0.1::double precision),
|
||||
('0.25', 0.25::double precision),
|
||||
('0.5', 0.5::double precision),
|
||||
('1', 1::double precision),
|
||||
('2.5', 2.5::double precision),
|
||||
('5', 5::double precision),
|
||||
('10', 10::double precision),
|
||||
('15', 15::double precision),
|
||||
('20', 20::double precision),
|
||||
('30', 30::double precision),
|
||||
('60', 60::double precision),
|
||||
('+Inf', NULL::double precision)
|
||||
)
|
||||
SELECT
|
||||
bounds.le,
|
||||
COUNT(events.job_id) FILTER (
|
||||
WHERE events.status = 'ready'
|
||||
AND (
|
||||
bounds.upper_bound IS NULL
|
||||
OR EXTRACT(EPOCH FROM (events.terminal_at - events.accepted_at)) <= bounds.upper_bound
|
||||
)
|
||||
)::bigint AS bucket
|
||||
FROM bounds
|
||||
LEFT JOIN attachment_terminal_events events ON events.status = 'ready'
|
||||
GROUP BY bounds.le, bounds.upper_bound
|
||||
ORDER BY bounds.upper_bound NULLS LAST
|
||||
metrics:
|
||||
- le:
|
||||
usage: LABEL
|
||||
description: Upper bound of the durable time-to-ready bucket in seconds.
|
||||
- bucket:
|
||||
usage: COUNTER
|
||||
description: Durable attachment jobs reaching ready within the bucket bound.
|
||||
|
||||
@@ -68,29 +68,29 @@ groups:
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)),
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)),
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.144
|
||||
for: 30s
|
||||
@@ -107,29 +107,29 @@ groups:
|
||||
expr: |
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)),
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
and
|
||||
(
|
||||
clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0))
|
||||
- (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)),
|
||||
0
|
||||
)
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0))
|
||||
)
|
||||
/ clamp_min(
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)),
|
||||
(sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0))
|
||||
+ (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)),
|
||||
0.000001
|
||||
) > 0.06
|
||||
for: 1m
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
|
||||
## Цель и сигналы
|
||||
|
||||
Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget.
|
||||
Объект контроля — первый terminal outcome каждой уникальной durable job. Цель — не менее 99% jobs впервые завершаются `ready` не позднее 30 секунд после acceptance за скользящие 30 дней. Первый `failed` и первое завершение `ready` позже 30 секунд расходуют error budget; replay не переписывает уже записанный outcome.
|
||||
|
||||
Основные сигналы:
|
||||
|
||||
- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker;
|
||||
- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы доставок worker;
|
||||
- `messenger_attachment_terminal_transitions_total{status}` — первые terminal outcomes из append-only PostgreSQL ledger;
|
||||
- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки;
|
||||
- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время;
|
||||
- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL;
|
||||
|
||||
Reference in New Issue
Block a user