From 3fde1c324c5c06f7506c7550e2bac3647dc336b2 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 9 Oct 2026 13:27:58 +0300 Subject: [PATCH] fix: harden attachment cleanup --- cmd/lab4experiment/main.go | 14 ++++---- internal/messenger/attachment/service.go | 7 ++-- internal/messenger/attachment/service_test.go | 23 ++++++++++++ internal/postgres/attachments.go | 33 ++++++++++++++--- internal/postgres/store_integration_test.go | 35 ++++++++++++++++++- internal/reconcile/reconcile.go | 19 +++++++--- internal/reconcile/reconcile_test.go | 18 +++++++--- migrations/0005_allow_pending_deletion.sql | 9 +++++ 8 files changed, 134 insertions(+), 24 deletions(-) create mode 100644 migrations/0005_allow_pending_deletion.sql diff --git a/cmd/lab4experiment/main.go b/cmd/lab4experiment/main.go index e8128b7..e263775 100644 --- a/cmd/lab4experiment/main.go +++ b/cmd/lab4experiment/main.go @@ -327,6 +327,9 @@ func runConsistency(arguments []string, output io.Writer) error { if err != nil { return err } + if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil { + return fmt.Errorf("prepare abandoned pending attachment: %w", err) + } keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID}) if err != nil { return err @@ -358,8 +361,7 @@ func runConsistency(arguments []string, output io.Writer) error { attachmentExists(ctx, pool, abandoned.ID) if _, err := pool.Exec(ctx, ` UPDATE attachments - SET created_at = clock_timestamp() - INTERVAL '2 hours', - ready_at = clock_timestamp() - INTERVAL '2 hours' + SET created_at = clock_timestamp() - INTERVAL '2 hours' WHERE id = $1`, abandoned.ID); err != nil { return fmt.Errorf("age abandoned attachment: %w", err) } @@ -394,7 +396,7 @@ func runConsistency(arguments []string, output io.Writer) error { FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult, FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey), UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID), - AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]), + AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]), TotalDurationMillis: time.Since(started).Milliseconds(), } if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted || @@ -414,7 +416,7 @@ func hasAnomalies(report reconcile.Report) bool { return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 || report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 || report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 || - report.OldUnreferencedAttachments != 0 + report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0 } func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) { @@ -580,11 +582,11 @@ func waitForReplicas(ctx context.Context, prometheusURL string, expected int) er } func activeReplicasQuery() string { - return `count(timestamp(process_resident_memory_bytes{job="messenger"}) > time() - 10)` + return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))` } func freshMetricQuery(metric string) string { - return fmt.Sprintf(`sum(%[1]s{job="messenger"} and on(instance) (timestamp(%[1]s{job="messenger"}) > time() - 10))`, metric) + return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric) } func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) { diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go index 3724d56..32f0179 100644 --- a/internal/messenger/attachment/service.go +++ b/internal/messenger/attachment/service.go @@ -11,6 +11,7 @@ import ( _ "image/png" "io" "log/slog" + "math" "mime" "net/http" "time" @@ -68,7 +69,7 @@ type Service struct { } func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) { - if config.MaxBytes < 1 || config.MaxPixels < 1 || config.CleanupTimeout <= 0 { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 { return nil, errors.New("invalid attachment service configuration") } if config.BodyMemoryBytes < config.MaxBytes { @@ -207,11 +208,11 @@ func (s *Service) readImage(input Upload) ([]byte, string, func(), error) { releaseBody() return nil, "", nil, model.ErrUnsupportedMedia } - pixels := int64(imageConfig.Width) * int64(imageConfig.Height) - if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > s.config.MaxPixels { + if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) { releaseBody() return nil, "", nil, model.ErrPayloadTooLarge } + pixels := int64(imageConfig.Width) * int64(imageConfig.Height) decodeBytes := pixels * 8 if !s.decodeMemory.TryAcquire(decodeBytes) { releaseBody() diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go index 3c5bb9b..fbaea3f 100644 --- a/internal/messenger/attachment/service_test.go +++ b/internal/messenger/attachment/service_test.go @@ -9,6 +9,7 @@ import ( "image/png" "io" "log/slog" + "math" "testing" "time" @@ -69,6 +70,28 @@ func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) { service.decodeMemory.Release(32) } +func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) { + oversized := pngFixture(t, 3, 2) + valid := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + + _, err := service.Upload(context.Background(), upload(oversized, "image/png")) + require.ErrorIs(t, err, model.ErrPayloadTooLarge) + _, err = service.Upload(context.Background(), upload(valid, "image/png")) + require.NoError(t, err) +} + +func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) { + _, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{}, + Config{ + MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024, + DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, + }, + ) + require.Error(t, err) +} + func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) { data := pngFixture(t, 2, 2) repository := &repositoryStub{} diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go index ae92171..305e806 100644 --- a/internal/postgres/attachments.go +++ b/internal/postgres/attachments.go @@ -161,32 +161,55 @@ func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) ( return referenced, nil } -func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, attachmentID string, createdBefore time.Time) (bool, error) { +func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) { tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) if err != nil { return false, fmt.Errorf("begin attachment deletion claim: %w", err) } defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + var status string var createdAt time.Time - err = tx.QueryRow(ctx, `SELECT created_at FROM attachments WHERE id = $1 FOR UPDATE`, attachmentID).Scan(&createdAt) + var readyAt *time.Time + err = tx.QueryRow( + ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID, + ).Scan(&status, &createdAt, &readyAt) if errors.Is(err, pgx.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("lock attachment deletion candidate: %w", err) } - if createdAt.After(createdBefore) { + if status != candidate.Status { + return false, nil + } + var age time.Time + switch status { + case model.AttachmentPending: + age = createdAt + case model.AttachmentReady: + if readyAt == nil { + return false, nil + } + age = *readyAt + default: + return false, nil + } + if age.After(createdBefore) { return false, nil } var referenced bool - if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, attachmentID).Scan(&referenced); err != nil { + if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil { return false, fmt.Errorf("recheck attachment references: %w", err) } if referenced { return false, nil } - if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1`, attachmentID); err != nil { + if _, err := tx.Exec( + ctx, + `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`, + candidate.ID, candidate.Status, + ); err != nil { return false, fmt.Errorf("mark attachment deleting: %w", err) } if err := tx.Commit(ctx); err != nil { diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go index 71943ea..48f6084 100644 --- a/internal/postgres/store_integration_test.go +++ b/internal/postgres/store_integration_test.go @@ -154,11 +154,44 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID) s.Require().NoError(err) s.True(referenced) - deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created.ID, time.Now().Add(time.Hour)) + deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour)) s.Require().NoError(err) s.False(deleted) } +func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + + claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.True(claimed) + deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID) + s.Require().NoError(err) + s.True(deleted) + + pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + + claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(claimed) + visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, visible.Status) +} + func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() { owner, firstChatID := createOwnerAndChat(s.T(), s.store) secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID) diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index d12a138..eb2b091 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -12,7 +12,7 @@ import ( type MetadataStore interface { ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) AttachmentExistsByObjectKey(context.Context, string) (bool, error) - ClaimAttachmentForDeletion(context.Context, string, time.Time) (bool, error) + ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error) DeleteAttachmentIfDeleting(context.Context, string) (bool, error) } @@ -36,6 +36,7 @@ type Report struct { Consistent int `json:"consistent"` FreshPendingAttachments int `json:"fresh_pending_attachments"` OldPendingAttachments int `json:"old_pending_attachments"` + DeletingAttachments int `json:"deleting_attachments"` FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"` OldUnreferencedAttachments int `json:"old_unreferenced_attachments"` DeletedAttachmentObjects int `json:"deleted_attachment_objects"` @@ -124,19 +125,21 @@ func (r *Reconciler) handlePair( ) error { report.MetadataScanned++ report.ObjectsScanned++ - report.Consistent++ switch metadata.Status { case model.AttachmentPending: if metadata.CreatedAt.After(anomalyCutoff) { report.FreshPendingAttachments++ + report.Consistent++ return nil } report.OldPendingAttachments++ return r.deletePair(ctx, metadata, anomalyCutoff, report) case model.AttachmentDeleting: + report.DeletingAttachments++ return r.deleteClaimedPair(ctx, metadata, report) case model.AttachmentReady: if metadata.Referenced { + report.Consistent++ return nil } age := metadata.CreatedAt @@ -145,6 +148,7 @@ func (r *Reconciler) handlePair( } if age.After(retentionCutoff) { report.FreshUnreferencedAttachments++ + report.Consistent++ return nil } report.OldUnreferencedAttachments++ @@ -158,7 +162,7 @@ func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment if !r.config.Apply { return nil } - claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment.ID, cutoff) + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff) if err != nil { return fmt.Errorf("claim attachment %q: %w", attachment.ID, err) } @@ -222,6 +226,9 @@ func (r *Reconciler) handleMissingObject( report *Report, ) error { report.MetadataScanned++ + if metadata.Status == model.AttachmentDeleting { + return r.deleteClaimedMetadata(ctx, metadata, report) + } if metadata.CreatedAt.After(cutoff) { report.FreshMissingObjects++ return nil @@ -234,13 +241,17 @@ func (r *Reconciler) handleMissingObject( if !r.config.Apply { return nil } - claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata.ID, cutoff) + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff) if err != nil { return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err) } if !claimed { return nil } + return r.deleteClaimedMetadata(ctx, metadata, report) +} + +func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error { deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID) if err != nil { return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err) diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 221ac0c..62471a6 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -121,13 +121,21 @@ func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string return exists, nil } -func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, id string, cutoff time.Time) (bool, error) { +func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) { for key, item := range s.items { - if item.ID == id && !item.CreatedAt.After(cutoff) && !item.Referenced { - item.Status = model.AttachmentDeleting - s.items[key] = item - return true, nil + if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced { + continue } + age := item.CreatedAt + if item.Status == model.AttachmentReady && item.ReadyAt != nil { + age = *item.ReadyAt + } + if age.After(cutoff) { + return false, nil + } + item.Status = model.AttachmentDeleting + s.items[key] = item + return true, nil } return false, nil } diff --git a/migrations/0005_allow_pending_deletion.sql b/migrations/0005_allow_pending_deletion.sql new file mode 100644 index 0000000..0b20acb --- /dev/null +++ b/migrations/0005_allow_pending_deletion.sql @@ -0,0 +1,9 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status = 'ready' AND ready_at IS NOT NULL) OR + status = 'deleting' + );