fix: harden attachment cleanup

This commit is contained in:
2026-10-09 13:27:58 +03:00
parent f8c31de7c0
commit 3fde1c324c
8 changed files with 134 additions and 24 deletions
+8 -6
View File
@@ -327,6 +327,9 @@ func runConsistency(arguments []string, output io.Writer) error {
if err != nil {
return err
}
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
}
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
if err != nil {
return err
@@ -358,8 +361,7 @@ func runConsistency(arguments []string, output io.Writer) error {
attachmentExists(ctx, pool, abandoned.ID)
if _, err := pool.Exec(ctx, `
UPDATE attachments
SET created_at = clock_timestamp() - INTERVAL '2 hours',
ready_at = clock_timestamp() - INTERVAL '2 hours'
SET created_at = clock_timestamp() - INTERVAL '2 hours'
WHERE id = $1`, abandoned.ID); err != nil {
return fmt.Errorf("age abandoned attachment: %w", err)
}
@@ -394,7 +396,7 @@ func runConsistency(arguments []string, output io.Writer) error {
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
TotalDurationMillis: time.Since(started).Milliseconds(),
}
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
@@ -414,7 +416,7 @@ func hasAnomalies(report reconcile.Report) bool {
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
report.OldUnreferencedAttachments != 0
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
}
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
@@ -580,11 +582,11 @@ func waitForReplicas(ctx context.Context, prometheusURL string, expected int) er
}
func activeReplicasQuery() string {
return `count(timestamp(process_resident_memory_bytes{job="messenger"}) > time() - 10)`
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
}
func freshMetricQuery(metric string) string {
return fmt.Sprintf(`sum(%[1]s{job="messenger"} and on(instance) (timestamp(%[1]s{job="messenger"}) > time() - 10))`, metric)
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
}
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
+4 -3
View File
@@ -11,6 +11,7 @@ import (
_ "image/png"
"io"
"log/slog"
"math"
"mime"
"net/http"
"time"
@@ -68,7 +69,7 @@ type Service struct {
}
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.CleanupTimeout <= 0 {
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
return nil, errors.New("invalid attachment service configuration")
}
if config.BodyMemoryBytes < config.MaxBytes {
@@ -207,11 +208,11 @@ func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
releaseBody()
return nil, "", nil, model.ErrUnsupportedMedia
}
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > s.config.MaxPixels {
if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) {
releaseBody()
return nil, "", nil, model.ErrPayloadTooLarge
}
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
decodeBytes := pixels * 8
if !s.decodeMemory.TryAcquire(decodeBytes) {
releaseBody()
@@ -9,6 +9,7 @@ import (
"image/png"
"io"
"log/slog"
"math"
"testing"
"time"
@@ -69,6 +70,28 @@ func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) {
service.decodeMemory.Release(32)
}
func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) {
oversized := pngFixture(t, 3, 2)
valid := pngFixture(t, 2, 2)
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
_, err := service.Upload(context.Background(), upload(oversized, "image/png"))
require.ErrorIs(t, err, model.ErrPayloadTooLarge)
_, err = service.Upload(context.Background(), upload(valid, "image/png"))
require.NoError(t, err)
}
func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
_, err := NewService(
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
Config{
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
},
)
require.Error(t, err)
}
func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
data := pngFixture(t, 2, 2)
repository := &repositoryStub{}
+28 -5
View File
@@ -161,32 +161,55 @@ func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (
return referenced, nil
}
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, attachmentID string, createdBefore time.Time) (bool, error) {
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return false, fmt.Errorf("begin attachment deletion claim: %w", err)
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
var status string
var createdAt time.Time
err = tx.QueryRow(ctx, `SELECT created_at FROM attachments WHERE id = $1 FOR UPDATE`, attachmentID).Scan(&createdAt)
var readyAt *time.Time
err = tx.QueryRow(
ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID,
).Scan(&status, &createdAt, &readyAt)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("lock attachment deletion candidate: %w", err)
}
if createdAt.After(createdBefore) {
if status != candidate.Status {
return false, nil
}
var age time.Time
switch status {
case model.AttachmentPending:
age = createdAt
case model.AttachmentReady:
if readyAt == nil {
return false, nil
}
age = *readyAt
default:
return false, nil
}
if age.After(createdBefore) {
return false, nil
}
var referenced bool
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, attachmentID).Scan(&referenced); err != nil {
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil {
return false, fmt.Errorf("recheck attachment references: %w", err)
}
if referenced {
return false, nil
}
if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1`, attachmentID); err != nil {
if _, err := tx.Exec(
ctx,
`UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`,
candidate.ID, candidate.Status,
); err != nil {
return false, fmt.Errorf("mark attachment deleting: %w", err)
}
if err := tx.Commit(ctx); err != nil {
+34 -1
View File
@@ -154,11 +154,44 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
s.Require().NoError(err)
s.True(referenced)
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created.ID, time.Now().Add(time.Hour))
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.False(deleted)
}
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
owner, chatID := createOwnerAndChat(s.T(), s.store)
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.True(claimed)
deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID)
s.Require().NoError(err)
s.True(deleted)
pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png",
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64),
ObjectKey: "attachments/" + uuid.NewString() + "/original",
})
s.Require().NoError(err)
ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
s.Require().NoError(err)
claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
s.Require().NoError(err)
s.False(claimed)
visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID)
s.Require().NoError(err)
s.Equal(model.AttachmentReady, visible.Status)
}
func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
+15 -4
View File
@@ -12,7 +12,7 @@ import (
type MetadataStore interface {
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
AttachmentExistsByObjectKey(context.Context, string) (bool, error)
ClaimAttachmentForDeletion(context.Context, string, time.Time) (bool, error)
ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error)
DeleteAttachmentIfDeleting(context.Context, string) (bool, error)
}
@@ -36,6 +36,7 @@ type Report struct {
Consistent int `json:"consistent"`
FreshPendingAttachments int `json:"fresh_pending_attachments"`
OldPendingAttachments int `json:"old_pending_attachments"`
DeletingAttachments int `json:"deleting_attachments"`
FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"`
OldUnreferencedAttachments int `json:"old_unreferenced_attachments"`
DeletedAttachmentObjects int `json:"deleted_attachment_objects"`
@@ -124,19 +125,21 @@ func (r *Reconciler) handlePair(
) error {
report.MetadataScanned++
report.ObjectsScanned++
report.Consistent++
switch metadata.Status {
case model.AttachmentPending:
if metadata.CreatedAt.After(anomalyCutoff) {
report.FreshPendingAttachments++
report.Consistent++
return nil
}
report.OldPendingAttachments++
return r.deletePair(ctx, metadata, anomalyCutoff, report)
case model.AttachmentDeleting:
report.DeletingAttachments++
return r.deleteClaimedPair(ctx, metadata, report)
case model.AttachmentReady:
if metadata.Referenced {
report.Consistent++
return nil
}
age := metadata.CreatedAt
@@ -145,6 +148,7 @@ func (r *Reconciler) handlePair(
}
if age.After(retentionCutoff) {
report.FreshUnreferencedAttachments++
report.Consistent++
return nil
}
report.OldUnreferencedAttachments++
@@ -158,7 +162,7 @@ func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment
if !r.config.Apply {
return nil
}
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment.ID, cutoff)
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff)
if err != nil {
return fmt.Errorf("claim attachment %q: %w", attachment.ID, err)
}
@@ -222,6 +226,9 @@ func (r *Reconciler) handleMissingObject(
report *Report,
) error {
report.MetadataScanned++
if metadata.Status == model.AttachmentDeleting {
return r.deleteClaimedMetadata(ctx, metadata, report)
}
if metadata.CreatedAt.After(cutoff) {
report.FreshMissingObjects++
return nil
@@ -234,13 +241,17 @@ func (r *Reconciler) handleMissingObject(
if !r.config.Apply {
return nil
}
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata.ID, cutoff)
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff)
if err != nil {
return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err)
}
if !claimed {
return nil
}
return r.deleteClaimedMetadata(ctx, metadata, report)
}
func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error {
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID)
if err != nil {
return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err)
+13 -5
View File
@@ -121,13 +121,21 @@ func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string
return exists, nil
}
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, id string, cutoff time.Time) (bool, error) {
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) {
for key, item := range s.items {
if item.ID == id && !item.CreatedAt.After(cutoff) && !item.Referenced {
item.Status = model.AttachmentDeleting
s.items[key] = item
return true, nil
if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced {
continue
}
age := item.CreatedAt
if item.Status == model.AttachmentReady && item.ReadyAt != nil {
age = *item.ReadyAt
}
if age.After(cutoff) {
return false, nil
}
item.Status = model.AttachmentDeleting
s.items[key] = item
return true, nil
}
return false, nil
}
@@ -0,0 +1,9 @@
ALTER TABLE attachments
DROP CONSTRAINT attachments_ready_at;
ALTER TABLE attachments
ADD CONSTRAINT attachments_ready_at CHECK (
(status = 'pending' AND ready_at IS NULL) OR
(status = 'ready' AND ready_at IS NOT NULL) OR
status = 'deleting'
);