forked from mtusi2026/mtusi2026lab4
fix: harden attachment cleanup
This commit is contained in:
@@ -327,6 +327,9 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
|
||||
}
|
||||
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -358,8 +361,7 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
attachmentExists(ctx, pool, abandoned.ID)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 hours',
|
||||
ready_at = clock_timestamp() - INTERVAL '2 hours'
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 hours'
|
||||
WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("age abandoned attachment: %w", err)
|
||||
}
|
||||
@@ -394,7 +396,7 @@ func runConsistency(arguments []string, output io.Writer) error {
|
||||
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
|
||||
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
|
||||
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
|
||||
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
||||
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
||||
TotalDurationMillis: time.Since(started).Milliseconds(),
|
||||
}
|
||||
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
|
||||
@@ -414,7 +416,7 @@ func hasAnomalies(report reconcile.Report) bool {
|
||||
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
|
||||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
|
||||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
|
||||
report.OldUnreferencedAttachments != 0
|
||||
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
|
||||
}
|
||||
|
||||
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
|
||||
@@ -580,11 +582,11 @@ func waitForReplicas(ctx context.Context, prometheusURL string, expected int) er
|
||||
}
|
||||
|
||||
func activeReplicasQuery() string {
|
||||
return `count(timestamp(process_resident_memory_bytes{job="messenger"}) > time() - 10)`
|
||||
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
|
||||
}
|
||||
|
||||
func freshMetricQuery(metric string) string {
|
||||
return fmt.Sprintf(`sum(%[1]s{job="messenger"} and on(instance) (timestamp(%[1]s{job="messenger"}) > time() - 10))`, metric)
|
||||
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
|
||||
}
|
||||
|
||||
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
_ "image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"mime"
|
||||
"net/http"
|
||||
"time"
|
||||
@@ -68,7 +69,7 @@ type Service struct {
|
||||
}
|
||||
|
||||
func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.CleanupTimeout <= 0 {
|
||||
if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 {
|
||||
return nil, errors.New("invalid attachment service configuration")
|
||||
}
|
||||
if config.BodyMemoryBytes < config.MaxBytes {
|
||||
@@ -207,11 +208,11 @@ func (s *Service) readImage(input Upload) ([]byte, string, func(), error) {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrUnsupportedMedia
|
||||
}
|
||||
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
|
||||
if imageConfig.Width < 1 || imageConfig.Height < 1 || pixels > s.config.MaxPixels {
|
||||
if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) {
|
||||
releaseBody()
|
||||
return nil, "", nil, model.ErrPayloadTooLarge
|
||||
}
|
||||
pixels := int64(imageConfig.Width) * int64(imageConfig.Height)
|
||||
decodeBytes := pixels * 8
|
||||
if !s.decodeMemory.TryAcquire(decodeBytes) {
|
||||
releaseBody()
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -69,6 +70,28 @@ func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) {
|
||||
service.decodeMemory.Release(32)
|
||||
}
|
||||
|
||||
func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) {
|
||||
oversized := pngFixture(t, 3, 2)
|
||||
valid := pngFixture(t, 2, 2)
|
||||
service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4)
|
||||
|
||||
_, err := service.Upload(context.Background(), upload(oversized, "image/png"))
|
||||
require.ErrorIs(t, err, model.ErrPayloadTooLarge)
|
||||
_, err = service.Upload(context.Background(), upload(valid, "image/png"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) {
|
||||
_, err := NewService(
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{},
|
||||
Config{
|
||||
MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024,
|
||||
DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second,
|
||||
},
|
||||
)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) {
|
||||
data := pngFixture(t, 2, 2)
|
||||
repository := &repositoryStub{}
|
||||
|
||||
@@ -161,32 +161,55 @@ func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (
|
||||
return referenced, nil
|
||||
}
|
||||
|
||||
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, attachmentID string, createdBefore time.Time) (bool, error) {
|
||||
func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("begin attachment deletion claim: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
var status string
|
||||
var createdAt time.Time
|
||||
err = tx.QueryRow(ctx, `SELECT created_at FROM attachments WHERE id = $1 FOR UPDATE`, attachmentID).Scan(&createdAt)
|
||||
var readyAt *time.Time
|
||||
err = tx.QueryRow(
|
||||
ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID,
|
||||
).Scan(&status, &createdAt, &readyAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("lock attachment deletion candidate: %w", err)
|
||||
}
|
||||
if createdAt.After(createdBefore) {
|
||||
if status != candidate.Status {
|
||||
return false, nil
|
||||
}
|
||||
var age time.Time
|
||||
switch status {
|
||||
case model.AttachmentPending:
|
||||
age = createdAt
|
||||
case model.AttachmentReady:
|
||||
if readyAt == nil {
|
||||
return false, nil
|
||||
}
|
||||
age = *readyAt
|
||||
default:
|
||||
return false, nil
|
||||
}
|
||||
if age.After(createdBefore) {
|
||||
return false, nil
|
||||
}
|
||||
var referenced bool
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, attachmentID).Scan(&referenced); err != nil {
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil {
|
||||
return false, fmt.Errorf("recheck attachment references: %w", err)
|
||||
}
|
||||
if referenced {
|
||||
return false, nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1`, attachmentID); err != nil {
|
||||
if _, err := tx.Exec(
|
||||
ctx,
|
||||
`UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`,
|
||||
candidate.ID, candidate.Status,
|
||||
); err != nil {
|
||||
return false, fmt.Errorf("mark attachment deleting: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
|
||||
@@ -154,11 +154,44 @@ func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() {
|
||||
referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID)
|
||||
s.Require().NoError(err)
|
||||
s.True(referenced)
|
||||
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created.ID, time.Now().Add(time.Hour))
|
||||
deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.False(deleted)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() {
|
||||
owner, chatID := createOwnerAndChat(s.T(), s.store)
|
||||
pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
|
||||
claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.True(claimed)
|
||||
deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID)
|
||||
s.Require().NoError(err)
|
||||
s.True(deleted)
|
||||
|
||||
pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{
|
||||
ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png",
|
||||
ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64),
|
||||
ObjectKey: "attachments/" + uuid.NewString() + "/original",
|
||||
})
|
||||
s.Require().NoError(err)
|
||||
ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey)
|
||||
s.Require().NoError(err)
|
||||
|
||||
claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour))
|
||||
s.Require().NoError(err)
|
||||
s.False(claimed)
|
||||
visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Equal(model.AttachmentReady, visible.Status)
|
||||
}
|
||||
|
||||
func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() {
|
||||
owner, firstChatID := createOwnerAndChat(s.T(), s.store)
|
||||
secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID)
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
type MetadataStore interface {
|
||||
ListAttachments(context.Context, string, int) ([]model.Attachment, string, error)
|
||||
AttachmentExistsByObjectKey(context.Context, string) (bool, error)
|
||||
ClaimAttachmentForDeletion(context.Context, string, time.Time) (bool, error)
|
||||
ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error)
|
||||
DeleteAttachmentIfDeleting(context.Context, string) (bool, error)
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ type Report struct {
|
||||
Consistent int `json:"consistent"`
|
||||
FreshPendingAttachments int `json:"fresh_pending_attachments"`
|
||||
OldPendingAttachments int `json:"old_pending_attachments"`
|
||||
DeletingAttachments int `json:"deleting_attachments"`
|
||||
FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"`
|
||||
OldUnreferencedAttachments int `json:"old_unreferenced_attachments"`
|
||||
DeletedAttachmentObjects int `json:"deleted_attachment_objects"`
|
||||
@@ -124,19 +125,21 @@ func (r *Reconciler) handlePair(
|
||||
) error {
|
||||
report.MetadataScanned++
|
||||
report.ObjectsScanned++
|
||||
report.Consistent++
|
||||
switch metadata.Status {
|
||||
case model.AttachmentPending:
|
||||
if metadata.CreatedAt.After(anomalyCutoff) {
|
||||
report.FreshPendingAttachments++
|
||||
report.Consistent++
|
||||
return nil
|
||||
}
|
||||
report.OldPendingAttachments++
|
||||
return r.deletePair(ctx, metadata, anomalyCutoff, report)
|
||||
case model.AttachmentDeleting:
|
||||
report.DeletingAttachments++
|
||||
return r.deleteClaimedPair(ctx, metadata, report)
|
||||
case model.AttachmentReady:
|
||||
if metadata.Referenced {
|
||||
report.Consistent++
|
||||
return nil
|
||||
}
|
||||
age := metadata.CreatedAt
|
||||
@@ -145,6 +148,7 @@ func (r *Reconciler) handlePair(
|
||||
}
|
||||
if age.After(retentionCutoff) {
|
||||
report.FreshUnreferencedAttachments++
|
||||
report.Consistent++
|
||||
return nil
|
||||
}
|
||||
report.OldUnreferencedAttachments++
|
||||
@@ -158,7 +162,7 @@ func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment.ID, cutoff)
|
||||
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("claim attachment %q: %w", attachment.ID, err)
|
||||
}
|
||||
@@ -222,6 +226,9 @@ func (r *Reconciler) handleMissingObject(
|
||||
report *Report,
|
||||
) error {
|
||||
report.MetadataScanned++
|
||||
if metadata.Status == model.AttachmentDeleting {
|
||||
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||
}
|
||||
if metadata.CreatedAt.After(cutoff) {
|
||||
report.FreshMissingObjects++
|
||||
return nil
|
||||
@@ -234,13 +241,17 @@ func (r *Reconciler) handleMissingObject(
|
||||
if !r.config.Apply {
|
||||
return nil
|
||||
}
|
||||
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata.ID, cutoff)
|
||||
claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err)
|
||||
}
|
||||
if !claimed {
|
||||
return nil
|
||||
}
|
||||
return r.deleteClaimedMetadata(ctx, metadata, report)
|
||||
}
|
||||
|
||||
func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error {
|
||||
deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err)
|
||||
|
||||
@@ -121,13 +121,21 @@ func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, id string, cutoff time.Time) (bool, error) {
|
||||
func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) {
|
||||
for key, item := range s.items {
|
||||
if item.ID == id && !item.CreatedAt.After(cutoff) && !item.Referenced {
|
||||
item.Status = model.AttachmentDeleting
|
||||
s.items[key] = item
|
||||
return true, nil
|
||||
if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced {
|
||||
continue
|
||||
}
|
||||
age := item.CreatedAt
|
||||
if item.Status == model.AttachmentReady && item.ReadyAt != nil {
|
||||
age = *item.ReadyAt
|
||||
}
|
||||
if age.After(cutoff) {
|
||||
return false, nil
|
||||
}
|
||||
item.Status = model.AttachmentDeleting
|
||||
s.items[key] = item
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT attachments_ready_at;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_ready_at CHECK (
|
||||
(status = 'pending' AND ready_at IS NULL) OR
|
||||
(status = 'ready' AND ready_at IS NOT NULL) OR
|
||||
status = 'deleting'
|
||||
);
|
||||
Reference in New Issue
Block a user