From 1ee4f8ececaaa48057578785cf1c59c389ef891e Mon Sep 17 00:00:00 2001 From: kk10 Date: Tue, 3 Nov 2026 15:12:55 +0300 Subject: [PATCH] feat(cleanup): remove stale queued attachments after grace period --- db/migrations/00003_attachments.sql | 3 +- db/queries/attachments.sql | 16 +++++- internal/app/app.go | 10 ++++ internal/app/cleanup.go | 79 ++++++++++++++++++++++++++++- internal/db/attachments.sql.go | 74 +++++++++++++++++++++++++-- internal/s3/client.go | 24 ++------- internal/service/attachment.go | 19 +++++-- 7 files changed, 189 insertions(+), 36 deletions(-) diff --git a/db/migrations/00003_attachments.sql b/db/migrations/00003_attachments.sql index 4a7b1af..febdea5 100644 --- a/db/migrations/00003_attachments.sql +++ b/db/migrations/00003_attachments.sql @@ -11,7 +11,7 @@ CREATE TABLE attachments ( content_type attachment_content_type NOT NULL, size_bytes int NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760), sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), - status attachment_status NOT NULL DEFAULT 'ready', + status attachment_status NOT NULL DEFAULT 'queued', error_code text, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now() @@ -19,6 +19,7 @@ CREATE TABLE attachments ( CREATE INDEX idx_attachments_chat_id ON attachments (chat_id, created_at); CREATE INDEX idx_attachments_owner_id ON attachments (owner_id); +CREATE INDEX idx_attachments_queued_created_at ON attachments (created_at) WHERE status = 'queued'; CREATE TABLE message_attachments ( message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, diff --git a/db/queries/attachments.sql b/db/queries/attachments.sql index ef74562..fd8d6f9 100644 --- a/db/queries/attachments.sql +++ b/db/queries/attachments.sql @@ -8,7 +8,6 @@ INSERT INTO attachments ( content_type, size_bytes, sha256, - status, error_code ) SELECT @@ -20,7 +19,6 @@ SELECT sqlc.arg(content_type)::attachment_content_type, sqlc.arg(size_bytes)::int, sqlc.arg(sha256)::text, - sqlc.arg(status)::attachment_status, NULLIF(sqlc.arg(error_code)::text, '') WHERE EXISTS ( SELECT 1 FROM chat_members @@ -61,3 +59,17 @@ SELECT message_id, attachment_id FROM message_attachments WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[]) ORDER BY message_id, position; + +-- name: ListStaleAttachmentUploads :many +SELECT id, s3_key FROM attachments +WHERE status = 'queued' AND created_at < sqlc.arg(cutoff)::timestamptz; + +-- name: MarkAttachmentReady :one +UPDATE attachments +SET status = 'ready', updated_at = now() +WHERE id = $1::uuid +RETURNING *; + +-- name: DeleteAttachment :exec +DELETE FROM attachments +WHERE id = $1::uuid; diff --git a/internal/app/app.go b/internal/app/app.go index ec2e889..b48c130 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -88,6 +88,16 @@ func Run() error { cleanupWG.Go(func() { runIdempotencyCleanup(cleanupCtx, log, queries, cfg.Server.IdempotencyTTL, idempotencyCleanupInterval) }) + cleanupWG.Go(func() { + runAttachmentCleanup( + cleanupCtx, + log, + queries, + s3Client, + attachmentOrphanGracePeriod, + attachmentCleanupInterval, + ) + }) defer func() { stopCleanup() cleanupWG.Wait() diff --git a/internal/app/cleanup.go b/internal/app/cleanup.go index d28b294..f0bbe51 100644 --- a/internal/app/cleanup.go +++ b/internal/app/cleanup.go @@ -4,10 +4,19 @@ import ( "context" "log/slog" "time" + + "github.com/google/uuid" + + "mtusi2026-messenger/internal/db" ) -const sessionCleanupInterval = time.Hour -const idempotencyCleanupInterval = time.Hour +const ( + sessionCleanupInterval = time.Hour + idempotencyCleanupInterval = time.Hour + + attachmentCleanupInterval = 10 * time.Minute + attachmentOrphanGracePeriod = time.Hour +) type sessionCleaner interface { DeleteExpiredSessions(ctx context.Context) (int64, error) @@ -17,6 +26,15 @@ type idempotencyCleaner interface { DeleteExpiredIdempotencyKeys(ctx context.Context, cutoff time.Time) (int64, error) } +type staleUploadLister interface { + ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]db.ListStaleAttachmentUploadsRow, error) + DeleteAttachment(ctx context.Context, id uuid.UUID) error +} + +type objectDeleter interface { + DeleteAdmin(ctx context.Context, key string) error +} + func runSessionCleanup(ctx context.Context, log *slog.Logger, q sessionCleaner, interval time.Duration) { ticker := time.NewTicker(interval) defer ticker.Stop() @@ -68,3 +86,60 @@ func runIdempotencyCleanup( } } } + +func runAttachmentCleanup( + ctx context.Context, + log *slog.Logger, + q staleUploadLister, + store objectDeleter, + grace time.Duration, + interval time.Duration, +) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cleanupStaleUploads(ctx, log, q, store, grace) + } + } +} + +func cleanupStaleUploads( + ctx context.Context, + log *slog.Logger, + q staleUploadLister, + store objectDeleter, + grace time.Duration, +) { + cutoff := time.Now().Add(-grace) + + rows, err := q.ListStaleAttachmentUploads(ctx, cutoff) + if err != nil { + if ctx.Err() == nil { + log.Error("list stale attachment uploads", "error", err) + } + return + } + + deleted := 0 + + for _, row := range rows { + if err := store.DeleteAdmin(ctx, row.S3Key); err != nil { + log.Error("delete stale upload object", "key", row.S3Key, "error", err) + continue + } + if err := q.DeleteAttachment(ctx, row.ID); err != nil { + log.Error("delete stale upload row", "id", row.ID, "error", err) + continue + } + deleted++ + } + + if deleted > 0 { + log.Info("stale attachment uploads deleted", "count", deleted) + } +} diff --git a/internal/db/attachments.sql.go b/internal/db/attachments.sql.go index 8ba1cba..843941c 100644 --- a/internal/db/attachments.sql.go +++ b/internal/db/attachments.sql.go @@ -7,6 +7,7 @@ package db import ( "context" + "time" "github.com/google/uuid" ) @@ -21,7 +22,6 @@ INSERT INTO attachments ( content_type, size_bytes, sha256, - status, error_code ) SELECT @@ -33,8 +33,7 @@ SELECT $6::attachment_content_type, $7::int, $8::text, - $9::attachment_status, - NULLIF($10::text, '') + NULLIF($9::text, '') WHERE EXISTS ( SELECT 1 FROM chat_members WHERE chat_id = $2 @@ -52,7 +51,6 @@ type CreateAttachmentParams struct { ContentType AttachmentContentType SizeBytes int Sha256 string - Status AttachmentStatus ErrorCode string } @@ -66,7 +64,6 @@ func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentPara arg.ContentType, arg.SizeBytes, arg.Sha256, - arg.Status, arg.ErrorCode, ) var i Attachment @@ -110,6 +107,16 @@ func (q *Queries) CreateMessageAttachment(ctx context.Context, arg CreateMessage return err } +const deleteAttachment = `-- name: DeleteAttachment :exec +DELETE FROM attachments +WHERE id = $1::uuid +` + +func (q *Queries) DeleteAttachment(ctx context.Context, dollar_1 uuid.UUID) error { + _, err := q.db.Exec(ctx, deleteAttachment, dollar_1) + return err +} + const getAttachmentByID = `-- name: GetAttachmentByID :one SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at FROM attachments WHERE id = $1::uuid @@ -225,3 +232,60 @@ func (q *Queries) ListMessageAttachmentsByMessageIDs(ctx context.Context, messag } return items, nil } + +const listStaleAttachmentUploads = `-- name: ListStaleAttachmentUploads :many +SELECT id, s3_key FROM attachments +WHERE status = 'queued' AND created_at < $1::timestamptz +` + +type ListStaleAttachmentUploadsRow struct { + ID uuid.UUID + S3Key string +} + +func (q *Queries) ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]ListStaleAttachmentUploadsRow, error) { + rows, err := q.db.Query(ctx, listStaleAttachmentUploads, cutoff) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListStaleAttachmentUploadsRow{} + for rows.Next() { + var i ListStaleAttachmentUploadsRow + if err := rows.Scan(&i.ID, &i.S3Key); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markAttachmentReady = `-- name: MarkAttachmentReady :one +UPDATE attachments +SET status = 'ready', updated_at = now() +WHERE id = $1::uuid +RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at +` + +func (q *Queries) MarkAttachmentReady(ctx context.Context, dollar_1 uuid.UUID) (Attachment, error) { + row := q.db.QueryRow(ctx, markAttachmentReady, dollar_1) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.S3Key, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.ErrorCode, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/s3/client.go b/internal/s3/client.go index b24884b..5f94e4a 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -24,7 +24,6 @@ var ( ErrObjectNotFound = errors.New("object not found") ErrPutObject = errors.New("put object") ErrGetObject = errors.New("get object") - ErrHeadObject = errors.New("head object") ErrDeleteObject = errors.New("delete object") ) @@ -145,25 +144,8 @@ func (c *Client) Get(ctx context.Context, key string) (*awss3.GetObjectOutput, e return out, nil } -func (c *Client) Head(ctx context.Context, key string) (*awss3.HeadObjectOutput, error) { - out, err := c.api.HeadObject(ctx, &awss3.HeadObjectInput{ - Bucket: aws.String(c.bucket), - Key: aws.String(key), - }) - if isNotFound(err) { - c.log.Warn("head object", "bucket", c.bucket, "key", key, "error", err) - return nil, ErrObjectNotFound - } - if err != nil { - c.log.Error("head object", "bucket", c.bucket, "key", key, "error", err) - return nil, ErrHeadObject - } - - return out, nil -} - -func (c *Client) Delete(ctx context.Context, key string) error { - _, err := c.api.DeleteObject(ctx, &awss3.DeleteObjectInput{ +func (c *Client) DeleteAdmin(ctx context.Context, key string) error { + _, err := c.bootstrap.DeleteObject(ctx, &awss3.DeleteObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(key), }) @@ -171,7 +153,7 @@ func (c *Client) Delete(ctx context.Context, key string) error { return nil } if err != nil { - c.log.Error("delete object", "bucket", c.bucket, "key", key, "error", err) + c.log.Error("delete admin object", "bucket", c.bucket, "key", key, "error", err) return ErrDeleteObject } diff --git a/internal/service/attachment.go b/internal/service/attachment.go index 59ab228..39de442 100644 --- a/internal/service/attachment.go +++ b/internal/service/attachment.go @@ -30,6 +30,7 @@ type attachmentQueries interface { CreateAttachment(ctx context.Context, arg db.CreateAttachmentParams) (db.Attachment, error) GetAttachmentByID(ctx context.Context, id uuid.UUID) (db.Attachment, error) IsChatMember(ctx context.Context, arg db.IsChatMemberParams) (bool, error) + MarkAttachmentReady(ctx context.Context, id uuid.UUID) (db.Attachment, error) } type AttachmentService struct { @@ -104,11 +105,8 @@ func (s *AttachmentService) Upload( if _, err := tmp.Seek(0, io.SeekStart); err != nil { return model.Attachment{}, err } - if err := s.s3.Put(ctx, key, tmp, size, string(detectedType)); err != nil { - return model.Attachment{}, err - } - row, err := s.q.CreateAttachment(ctx, db.CreateAttachmentParams{ + _, err = s.q.CreateAttachment(ctx, db.CreateAttachmentParams{ ID: id, ChatID: chatID, OwnerID: userID, @@ -117,9 +115,20 @@ func (s *AttachmentService) Upload( ContentType: db.AttachmentContentType(detectedType), SizeBytes: int(size), Sha256: hash, - Status: db.AttachmentStatusReady, ErrorCode: "", }) + if postgres.IsNotFound(err) { + return model.Attachment{}, ErrAttachmentNotFound + } + if err != nil { + return model.Attachment{}, err + } + + if err := s.s3.Put(ctx, key, tmp, size, string(detectedType)); err != nil { + return model.Attachment{}, err + } + + row, err := s.q.MarkAttachmentReady(ctx, id) if err != nil { return model.Attachment{}, err }