From f23855f78605d30865ced3bb6265e4c95b0692c6 Mon Sep 17 00:00:00 2001 From: dizi Date: Wed, 4 Nov 2026 20:56:35 +0300 Subject: [PATCH] fix: persist attachment SLO outcomes --- .../messenger/attachment/pipeline/metrics.go | 13 +- .../messenger/attachment/worker/worker.go | 5 +- internal/postgres/attachment_jobs.go | 38 ++-- internal/postgres/attachment_recovery.go | 11 +- internal/postgres/store_integration_test.go | 214 ++++++++++++++++++ .../0008_attachment_terminal_events.sql | 33 +++ monitoring/postgres-queries.yml | 58 +++++ monitoring/rules/messenger.yml | 32 +-- monitoring/runbooks/attachment-pipeline.md | 5 +- 9 files changed, 357 insertions(+), 52 deletions(-) create mode 100644 migrations/0008_attachment_terminal_events.sql diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go index 07a91a4..3bc4ef7 100644 --- a/internal/messenger/attachment/pipeline/metrics.go +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -17,7 +17,6 @@ type Metrics struct { workerInFlight metric.Int64UpDownCounter workerJobs metric.Int64Counter workerDuration metric.Float64Histogram - workerTimeToReady metric.Float64Histogram workerRetries metric.Int64Counter workerReclaimed metric.Int64Counter outboxPublished metric.Int64Counter @@ -39,10 +38,6 @@ func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *M metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(workerDurationBuckets...), ) - workerTimeToReady, _ := meter.Float64Histogram("messenger_attachment_time_to_ready", - metric.WithDescription("Elapsed time from durable job creation to ready state."), metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(workerDurationBuckets...), - ) workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt.")) workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout.")) outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams.")) @@ -59,8 +54,8 @@ func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *M } return &Metrics{ meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs, - workerDuration: workerDuration, workerTimeToReady: workerTimeToReady, - workerRetries: workerRetries, workerReclaimed: workerReclaimed, + workerDuration: workerDuration, + workerRetries: workerRetries, workerReclaimed: workerReclaimed, outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure, outboxRecovered: outboxRecovered, handler: handler, } @@ -81,10 +76,6 @@ func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration t m.workerDuration.Record(ctx, duration.Seconds(), attributes) } -func (m *Metrics) ObserveTimeToReady(ctx context.Context, duration time.Duration) { - m.workerTimeToReady.Record(ctx, duration.Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process"))) -} - func (m *Metrics) ObserveRetry(ctx context.Context) { m.workerRetries.Add(ctx, 1) } diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go index 45ff600..8b88c84 100644 --- a/internal/messenger/attachment/worker/worker.go +++ b/internal/messenger/attachment/worker/worker.go @@ -58,7 +58,6 @@ type ImageProcessor interface { type Metrics interface { ObserveStarted(context.Context) ObserveFinished(context.Context, string, time.Duration) - ObserveTimeToReady(context.Context, time.Duration) ObserveRetry(context.Context) ObserveReclaimed(context.Context, int) } @@ -290,7 +289,6 @@ func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Deliv return err } result = "ready" - w.metrics.ObserveTimeToReady(ctx, w.now().Sub(envelope.CreatedAt)) return nil } @@ -350,7 +348,7 @@ func (w *Worker) resolveClaim( if job.ErrorCode != nil { code = *job.ErrorCode } - return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + return "duplicate_failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) default: return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) } @@ -538,6 +536,5 @@ type nopMetrics struct{} func (nopMetrics) ObserveStarted(context.Context) {} func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {} -func (nopMetrics) ObserveTimeToReady(context.Context, time.Duration) {} func (nopMetrics) ObserveRetry(context.Context) {} func (nopMetrics) ObserveReclaimed(context.Context, int) {} diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go index 6c959de..0442e4d 100644 --- a/internal/postgres/attachment_jobs.go +++ b/internal/postgres/attachment_jobs.go @@ -37,9 +37,6 @@ func (s *Store) QueueAttachmentWithJobAndOutbox( return model.Attachment{}, model.ErrInvalidInput } now := time.Now().UTC() - if job.CreatedAt.IsZero() { - job.CreatedAt = now - } if job.AvailableAt.IsZero() { job.AvailableAt = now } @@ -67,8 +64,7 @@ func (s *Store) QueueAttachmentWithJobAndOutbox( job.AttachmentID = attachmentID job.Status = model.AttachmentJobQueued - job.UpdatedAt = job.CreatedAt - if err := insertAttachmentJob(ctx, tx, job); err != nil { + if err := insertAttachmentJob(ctx, tx, &job); err != nil { return model.Attachment{}, err } if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { @@ -92,7 +88,6 @@ func (s *Store) QueueAttachmentPipelineVersion( } job.Attempt = 1 now := time.Now().UTC() - job.CreatedAt = now job.AvailableAt = now tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) @@ -114,7 +109,6 @@ func (s *Store) QueueAttachmentPipelineVersion( } job.AttachmentID = attachment.ID job.Status = model.AttachmentJobQueued - job.UpdatedAt = now const queueAttachment = ` UPDATE attachments SET status = 'queued', pipeline_version = $2, staging_key = object_key, @@ -124,7 +118,7 @@ func (s *Store) QueueAttachmentPipelineVersion( if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil { return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err) } - if err := insertAttachmentJob(ctx, tx, job); err != nil { + if err := insertAttachmentJob(ctx, tx, &job); err != nil { return model.Attachment{}, err } if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { @@ -163,15 +157,16 @@ func (s *Store) alreadyQueuedAttachment( return publicAttachment(attachment), nil } -func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job model.AttachmentJob) error { +func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job *model.AttachmentJob) error { const query = ` INSERT INTO attachment_jobs ( id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at ) - VALUES ($1, $2, $3, $4, 'queued', $5, $6, $6)` - if _, err := tx.Exec( - ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, job.CreatedAt, - ); err != nil { + VALUES ($1, $2, $3, $4, 'queued', $5, statement_timestamp(), statement_timestamp()) + RETURNING created_at, updated_at` + if err := tx.QueryRow( + ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, + ).Scan(&job.CreatedAt, &job.UpdatedAt); err != nil { return fmt.Errorf("insert attachment job: %w", err) } return nil @@ -417,6 +412,9 @@ func (s *Store) CompleteAttachmentJob( if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil { return "", fmt.Errorf("complete attachment: %w", err) } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobReady); err != nil { + return "", err + } if err := tx.Commit(ctx); err != nil { return "", fmt.Errorf("commit completed attachment job: %w", err) } @@ -527,12 +525,26 @@ func (s *Store) FailAttachmentJob( if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil { return "", fmt.Errorf("fail attachment: %w", err) } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil { + return "", err + } if err := tx.Commit(ctx); err != nil { return "", fmt.Errorf("commit failed attachment job: %w", err) } return model.JobMutationApplied, nil } +func insertAttachmentTerminalEvent(ctx context.Context, tx pgx.Tx, job model.AttachmentJob, status string) error { + const query = ` + INSERT INTO attachment_terminal_events (job_id, status, accepted_at) + VALUES ($1, $2, $3) + ON CONFLICT (job_id) DO NOTHING` + if _, err := tx.Exec(ctx, query, job.ID, status, job.CreatedAt); err != nil { + return fmt.Errorf("insert attachment terminal event: %w", err) + } + return nil +} + func lockJobMutation( ctx context.Context, tx pgx.Tx, diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go index e305a79..388dafc 100644 --- a/internal/postgres/attachment_recovery.go +++ b/internal/postgres/attachment_recovery.go @@ -89,9 +89,7 @@ func (s *Store) RecoverExpiredAttachmentJobs( if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil { return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err) } - eventJob := job - eventJob.CreatedAt = time.Now().UTC() - if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil { return model.RecoveryResult{}, err } recovered.Requeued++ @@ -118,6 +116,9 @@ func markExpiredJobFailed( if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil { return fmt.Errorf("fail expired attachment job: %w", err) } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil { + return err + } if !failAttachment { return nil } @@ -198,9 +199,7 @@ func (s *Store) ReplayFailedAttachment( ); err != nil { return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err) } - eventJob := job - eventJob.CreatedAt = time.Now().UTC() - if err := insertAttachmentJobOutbox(ctx, tx, eventJob, "", ""); err != nil { + if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil { return model.AttachmentJob{}, err } if err := tx.Commit(ctx); err != nil { diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index e4184db..e71ab6b 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/suite" @@ -19,6 +20,7 @@ import ( "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" ) type PostgresSuite struct { @@ -210,6 +212,7 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants) s.Require().NoError(err) s.Equal(model.JobMutationApplied, result) + s.assertTerminalEvent(job.ID, model.AttachmentJobReady, lease.Job.CreatedAt) result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) s.Require().NoError(err) s.Equal(model.JobMutationAlreadyComplete, result) @@ -252,6 +255,130 @@ func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { s.Equal(model.JobMutationStale, result) } +func (s *PostgresSuite) TestAttachmentJobCreationUsesDatabaseClock() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/original" + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: owner.ID, Filename: "future.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("f", 64), ObjectKey: stagingKey, + }) + s.Require().NoError(err) + future := time.Now().Add(24 * time.Hour) + job := model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, + Attempt: 1, CreatedAt: future, + } + _, err = s.store.QueueAttachmentWithJobAndOutbox(context.Background(), pending.ID, stagingKey, job, "", "") + s.Require().NoError(err) + stored, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.True(stored.CreatedAt.Before(future)) + + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachmentID, 1, 1, "worker-clock", time.Minute, + ) + s.Require().NoError(err) + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachmentID, 1)) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + s.assertTerminalEvent(job.ID, model.AttachmentJobReady, stored.CreatedAt) + s.assertOutboxCreationTime(job.ID, 1, stored.CreatedAt) +} + +func (s *PostgresSuite) TestTerminalLedgerMigrationBackfillsExistingJob() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + replayedAttachment, replayedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + skewedAttachment, skewedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + replayedAcceptedAt := time.Now().Add(-2 * time.Minute).UTC().Truncate(time.Microsecond) + firstFailureAt := replayedAcceptedAt.Add(20 * time.Second) + replayedReadyAt := firstFailureAt.Add(time.Minute) + skewedTerminalAt := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond) + skewedCreatedAt := skewedTerminalAt.Add(time.Minute) + tx, err := s.pool.Begin(context.Background()) + s.Require().NoError(err) + defer func() { s.Require().NoError(tx.Rollback(context.Background())) }() + _, err = tx.Exec( + context.Background(), + `UPDATE attachment_jobs SET status = 'ready', attempt = 2, error_code = NULL, created_at = $2, updated_at = $3 + WHERE id = $1`, + replayedJob.ID, + replayedAcceptedAt, + replayedReadyAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = $2 + WHERE id = $1`, + replayedAttachment.ID, + replayedReadyAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `INSERT INTO attachment_replay_audit ( + id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason, created_at + ) VALUES ($1, $2, $3, 1, 2, 'legacy-operator', 'legacy replay', $4)`, + uuid.NewString(), + replayedJob.ID, + replayedAttachment.ID, + firstFailureAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachment_jobs SET status = 'failed', error_code = 'legacy_failure', created_at = $2, updated_at = $3 + WHERE id = $1`, + skewedJob.ID, + skewedCreatedAt, + skewedTerminalAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachments SET status = 'failed', error_code = 'legacy_failure', ready_at = NULL + WHERE id = $1`, + skewedAttachment.ID, + ) + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), `ALTER INDEX attachment_terminal_events_status_terminal_at_idx RENAME TO attachment_terminal_events_status_terminal_at_idx_old`) + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), `ALTER TABLE attachment_terminal_events RENAME TO attachment_terminal_events_old`) + s.Require().NoError(err) + migration, err := migrations.FS.ReadFile("0008_attachment_terminal_events.sql") + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), string(migration), pgx.QueryExecModeSimpleProtocol) + s.Require().NoError(err) + + s.assertTerminalEventInTx(tx, replayedJob.ID, model.AttachmentJobFailed, replayedAcceptedAt, firstFailureAt) + s.assertTerminalEventInTx(tx, skewedJob.ID, model.AttachmentJobFailed, skewedTerminalAt, skewedTerminalAt) +} + +func (s *PostgresSuite) assertTerminalEventInTx( + tx pgx.Tx, + jobID string, + status string, + acceptedAt time.Time, + terminalAt time.Time, +) { + s.T().Helper() + var ( + actualStatus string + actualAcceptedAt time.Time + actualTerminalAt time.Time + ) + err := tx.QueryRow( + context.Background(), + `SELECT status, accepted_at, terminal_at FROM attachment_terminal_events WHERE job_id = $1`, + jobID, + ).Scan(&actualStatus, &actualAcceptedAt, &actualTerminalAt) + s.Require().NoError(err) + s.Equal(status, actualStatus) + s.True(acceptedAt.Equal(actualAcceptedAt)) + s.True(terminalAt.Equal(actualTerminalAt)) +} + func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { owner, chatID := createOwnerAndChat(s.T(), s.store) _, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) @@ -295,6 +422,8 @@ func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { owner, chatID := createOwnerAndChat(s.T(), s.store) attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) lease, err := s.store.ClaimAttachmentJob( context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, ) @@ -302,6 +431,7 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image") s.Require().NoError(err) s.Equal(model.JobMutationApplied, result) + firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt) _, _, err = s.store.CreateMessage( context.Background(), chatID, owner.ID, "failed attachment", @@ -320,6 +450,18 @@ func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { s.Require().NoError(err) s.Equal(2, replayed.Attempt) s.Equal(model.AttachmentJobQueued, replayed.Status) + s.Equal(original.CreatedAt, replayed.CreatedAt) + s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt) + replayedLease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, replayed.PipelineVersion, replayed.Attempt, "worker-2", time.Minute, + ) + s.Require().NoError(err) + result, err = s.store.CompleteAttachmentJob( + context.Background(), replayedLease, completedVariants(attachment.ID, replayed.PipelineVersion), + ) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)) var auditCount int err = s.pool.QueryRow( @@ -336,6 +478,8 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, ) s.Require().NoError(err) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) _, err = s.pool.Exec( context.Background(), `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, @@ -351,12 +495,82 @@ func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { s.Require().NoError(err) s.Equal(2, current.Attempt) s.Equal(model.AttachmentJobQueued, current.Status) + s.Equal(original.CreatedAt, current.CreatedAt) + s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt) result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1)) s.Require().NoError(err) s.Equal(model.JobMutationStale, result) } +func (s *PostgresSuite) TestExpiredLeaseRecoveryFailsExhaustedJobOnce() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + _, err = s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + _, err = s.pool.Exec( + context.Background(), + `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, + job.ID, + ) + s.Require().NoError(err) + + recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now()) + s.Require().NoError(err) + s.Zero(recovered.Requeued) + s.Equal(1, recovered.Failed) + firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt) + + recovered, err = s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now()) + s.Require().NoError(err) + s.Zero(recovered.Requeued) + s.Zero(recovered.Failed) + current, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentJobFailed, current.Status) + s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)) +} + +func (s *PostgresSuite) assertTerminalEvent(jobID string, status string, acceptedAt time.Time) time.Time { + s.T().Helper() + var ( + actualStatus string + actualAcceptedAt time.Time + terminalAt time.Time + count int + ) + err := s.pool.QueryRow( + context.Background(), + `SELECT min(status), min(accepted_at), min(terminal_at), count(*) + FROM attachment_terminal_events WHERE job_id = $1`, + jobID, + ).Scan(&actualStatus, &actualAcceptedAt, &terminalAt, &count) + s.Require().NoError(err) + s.Equal(1, count) + s.Equal(status, actualStatus) + s.True(acceptedAt.Equal(actualAcceptedAt), "expected %s, got %s", acceptedAt, actualAcceptedAt) + return terminalAt +} + +func (s *PostgresSuite) assertOutboxCreationTime(jobID string, attempt int, expected time.Time) { + s.T().Helper() + var payload []byte + err := s.pool.QueryRow( + context.Background(), + `SELECT payload FROM outbox_events WHERE job_id = $1 AND attempt = $2`, + jobID, + attempt, + ).Scan(&payload) + s.Require().NoError(err) + var message model.AttachmentJobMessage + s.Require().NoError(json.Unmarshal(payload, &message)) + s.True(expected.Equal(message.CreatedAt), "expected %s, got %s", expected, message.CreatedAt) +} + func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() { owner, chatID := createOwnerAndChat(s.T(), s.store) attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) diff --git a/migrations/0008_attachment_terminal_events.sql b/migrations/0008_attachment_terminal_events.sql new file mode 100644 index 0000000..53f17ce --- /dev/null +++ b/migrations/0008_attachment_terminal_events.sql @@ -0,0 +1,33 @@ +CREATE TABLE attachment_terminal_events ( + job_id UUID PRIMARY KEY, + status TEXT NOT NULL, + accepted_at TIMESTAMPTZ NOT NULL, + terminal_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_terminal_events_status CHECK (status IN ('ready', 'failed')), + CONSTRAINT attachment_terminal_events_time CHECK (terminal_at >= accepted_at) +); + +CREATE INDEX attachment_terminal_events_status_terminal_at_idx + ON attachment_terminal_events (status, terminal_at); + +WITH first_replay AS ( + SELECT job_id, MIN(created_at) AS created_at + FROM attachment_replay_audit + GROUP BY job_id +) +INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at) +SELECT jobs.id, 'failed', LEAST(jobs.created_at, first_replay.created_at), first_replay.created_at +FROM attachment_jobs jobs +JOIN first_replay ON first_replay.job_id = jobs.id +ON CONFLICT (job_id) DO NOTHING; + +INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at) +SELECT jobs.id, jobs.status, LEAST(jobs.created_at, jobs.updated_at), jobs.updated_at +FROM attachment_jobs jobs +WHERE jobs.status IN ('ready', 'failed') + AND NOT EXISTS ( + SELECT 1 + FROM attachment_replay_audit replay + WHERE replay.job_id = jobs.id + ) +ON CONFLICT (job_id) DO NOTHING; diff --git a/monitoring/postgres-queries.yml b/monitoring/postgres-queries.yml index 7666281..52fcfe1 100644 --- a/monitoring/postgres-queries.yml +++ b/monitoring/postgres-queries.yml @@ -37,3 +37,61 @@ messenger_outbox: - oldest_pending_age_seconds: usage: GAUGE description: Age of the oldest unpublished outbox event. + +messenger_attachment_terminal_transitions: + query: | + WITH statuses(status) AS ( + VALUES ('ready'), ('failed') + ) + SELECT + statuses.status, + COUNT(events.job_id)::bigint AS total + FROM statuses + LEFT JOIN attachment_terminal_events events ON events.status = statuses.status + GROUP BY statuses.status + metrics: + - status: + usage: LABEL + description: First terminal status recorded for a durable attachment job. + - total: + usage: COUNTER + description: Durable attachment jobs whose first terminal outcome has this status. + +messenger_attachment_time_to_ready_seconds: + query: | + WITH bounds(le, upper_bound) AS ( + VALUES + ('0.05', 0.05::double precision), + ('0.1', 0.1::double precision), + ('0.25', 0.25::double precision), + ('0.5', 0.5::double precision), + ('1', 1::double precision), + ('2.5', 2.5::double precision), + ('5', 5::double precision), + ('10', 10::double precision), + ('15', 15::double precision), + ('20', 20::double precision), + ('30', 30::double precision), + ('60', 60::double precision), + ('+Inf', NULL::double precision) + ) + SELECT + bounds.le, + COUNT(events.job_id) FILTER ( + WHERE events.status = 'ready' + AND ( + bounds.upper_bound IS NULL + OR EXTRACT(EPOCH FROM (events.terminal_at - events.accepted_at)) <= bounds.upper_bound + ) + )::bigint AS bucket + FROM bounds + LEFT JOIN attachment_terminal_events events ON events.status = 'ready' + GROUP BY bounds.le, bounds.upper_bound + ORDER BY bounds.upper_bound NULLS LAST + metrics: + - le: + usage: LABEL + description: Upper bound of the durable time-to-ready bucket in seconds. + - bucket: + usage: COUNTER + description: Durable attachment jobs reaching ready within the bucket bound. diff --git a/monitoring/rules/messenger.yml b/monitoring/rules/messenger.yml index 4999ddb..78a86dd 100644 --- a/monitoring/rules/messenger.yml +++ b/monitoring/rules/messenger.yml @@ -68,29 +68,29 @@ groups: expr: | ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[5m])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[5m])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)), 0.000001 ) > 0.144 and ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[1h])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[1h])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)), 0.000001 ) > 0.144 for: 30s @@ -107,29 +107,29 @@ groups: expr: | ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[30m])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[30m])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)), 0.000001 ) > 0.06 and ( clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)), 0 ) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)) ) / clamp_min( - (sum(rate(messenger_attachment_time_to_ready_seconds_count[6h])) or vector(0)) - + (sum(rate(messenger_attachment_worker_jobs_total{result="failed"}[6h])) or vector(0)), + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)), 0.000001 ) > 0.06 for: 1m diff --git a/monitoring/runbooks/attachment-pipeline.md b/monitoring/runbooks/attachment-pipeline.md index c2ae007..4c7155f 100644 --- a/monitoring/runbooks/attachment-pipeline.md +++ b/monitoring/runbooks/attachment-pipeline.md @@ -2,11 +2,12 @@ ## Цель и сигналы -Объект контроля — время от durable acceptance изображения до состояния `ready`. Цель — не менее 99% jobs завершаются `ready` не позднее 30 секунд за скользящие 30 дней. `failed` и завершение позже 30 секунд расходуют error budget. +Объект контроля — первый terminal outcome каждой уникальной durable job. Цель — не менее 99% jobs впервые завершаются `ready` не позднее 30 секунд после acceptance за скользящие 30 дней. Первый `failed` и первое завершение `ready` позже 30 секунд расходуют error budget; replay не переписывает уже записанный outcome. Основные сигналы: -- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы worker; +- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы доставок worker; +- `messenger_attachment_terminal_transitions_total{status}` — первые terminal outcomes из append-only PostgreSQL ledger; - `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки; - `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время; - `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL;