feat(cleanup): remove stale queued attachments after grace period

This commit is contained in:
kk10
2026-11-03 15:12:55 +03:00
parent fbd9c56b1f
commit 1ee4f8ecec
7 changed files with 189 additions and 36 deletions
+2 -1
View File
@@ -11,7 +11,7 @@ CREATE TABLE attachments (
content_type attachment_content_type NOT NULL,
size_bytes int NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
status attachment_status NOT NULL DEFAULT 'ready',
status attachment_status NOT NULL DEFAULT 'queued',
error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
@@ -19,6 +19,7 @@ CREATE TABLE attachments (
CREATE INDEX idx_attachments_chat_id ON attachments (chat_id, created_at);
CREATE INDEX idx_attachments_owner_id ON attachments (owner_id);
CREATE INDEX idx_attachments_queued_created_at ON attachments (created_at) WHERE status = 'queued';
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
+14 -2
View File
@@ -8,7 +8,6 @@ INSERT INTO attachments (
content_type,
size_bytes,
sha256,
status,
error_code
)
SELECT
@@ -20,7 +19,6 @@ SELECT
sqlc.arg(content_type)::attachment_content_type,
sqlc.arg(size_bytes)::int,
sqlc.arg(sha256)::text,
sqlc.arg(status)::attachment_status,
NULLIF(sqlc.arg(error_code)::text, '')
WHERE EXISTS (
SELECT 1 FROM chat_members
@@ -61,3 +59,17 @@ SELECT message_id, attachment_id
FROM message_attachments
WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
ORDER BY message_id, position;
-- name: ListStaleAttachmentUploads :many
SELECT id, s3_key FROM attachments
WHERE status = 'queued' AND created_at < sqlc.arg(cutoff)::timestamptz;
-- name: MarkAttachmentReady :one
UPDATE attachments
SET status = 'ready', updated_at = now()
WHERE id = $1::uuid
RETURNING *;
-- name: DeleteAttachment :exec
DELETE FROM attachments
WHERE id = $1::uuid;
+10
View File
@@ -88,6 +88,16 @@ func Run() error {
cleanupWG.Go(func() {
runIdempotencyCleanup(cleanupCtx, log, queries, cfg.Server.IdempotencyTTL, idempotencyCleanupInterval)
})
cleanupWG.Go(func() {
runAttachmentCleanup(
cleanupCtx,
log,
queries,
s3Client,
attachmentOrphanGracePeriod,
attachmentCleanupInterval,
)
})
defer func() {
stopCleanup()
cleanupWG.Wait()
+77 -2
View File
@@ -4,10 +4,19 @@ import (
"context"
"log/slog"
"time"
"github.com/google/uuid"
"mtusi2026-messenger/internal/db"
)
const sessionCleanupInterval = time.Hour
const idempotencyCleanupInterval = time.Hour
const (
sessionCleanupInterval = time.Hour
idempotencyCleanupInterval = time.Hour
attachmentCleanupInterval = 10 * time.Minute
attachmentOrphanGracePeriod = time.Hour
)
type sessionCleaner interface {
DeleteExpiredSessions(ctx context.Context) (int64, error)
@@ -17,6 +26,15 @@ type idempotencyCleaner interface {
DeleteExpiredIdempotencyKeys(ctx context.Context, cutoff time.Time) (int64, error)
}
type staleUploadLister interface {
ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]db.ListStaleAttachmentUploadsRow, error)
DeleteAttachment(ctx context.Context, id uuid.UUID) error
}
type objectDeleter interface {
DeleteAdmin(ctx context.Context, key string) error
}
func runSessionCleanup(ctx context.Context, log *slog.Logger, q sessionCleaner, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
@@ -68,3 +86,60 @@ func runIdempotencyCleanup(
}
}
}
func runAttachmentCleanup(
ctx context.Context,
log *slog.Logger,
q staleUploadLister,
store objectDeleter,
grace time.Duration,
interval time.Duration,
) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
cleanupStaleUploads(ctx, log, q, store, grace)
}
}
}
func cleanupStaleUploads(
ctx context.Context,
log *slog.Logger,
q staleUploadLister,
store objectDeleter,
grace time.Duration,
) {
cutoff := time.Now().Add(-grace)
rows, err := q.ListStaleAttachmentUploads(ctx, cutoff)
if err != nil {
if ctx.Err() == nil {
log.Error("list stale attachment uploads", "error", err)
}
return
}
deleted := 0
for _, row := range rows {
if err := store.DeleteAdmin(ctx, row.S3Key); err != nil {
log.Error("delete stale upload object", "key", row.S3Key, "error", err)
continue
}
if err := q.DeleteAttachment(ctx, row.ID); err != nil {
log.Error("delete stale upload row", "id", row.ID, "error", err)
continue
}
deleted++
}
if deleted > 0 {
log.Info("stale attachment uploads deleted", "count", deleted)
}
}
+69 -5
View File
@@ -7,6 +7,7 @@ package db
import (
"context"
"time"
"github.com/google/uuid"
)
@@ -21,7 +22,6 @@ INSERT INTO attachments (
content_type,
size_bytes,
sha256,
status,
error_code
)
SELECT
@@ -33,8 +33,7 @@ SELECT
$6::attachment_content_type,
$7::int,
$8::text,
$9::attachment_status,
NULLIF($10::text, '')
NULLIF($9::text, '')
WHERE EXISTS (
SELECT 1 FROM chat_members
WHERE chat_id = $2
@@ -52,7 +51,6 @@ type CreateAttachmentParams struct {
ContentType AttachmentContentType
SizeBytes int
Sha256 string
Status AttachmentStatus
ErrorCode string
}
@@ -66,7 +64,6 @@ func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentPara
arg.ContentType,
arg.SizeBytes,
arg.Sha256,
arg.Status,
arg.ErrorCode,
)
var i Attachment
@@ -110,6 +107,16 @@ func (q *Queries) CreateMessageAttachment(ctx context.Context, arg CreateMessage
return err
}
const deleteAttachment = `-- name: DeleteAttachment :exec
DELETE FROM attachments
WHERE id = $1::uuid
`
func (q *Queries) DeleteAttachment(ctx context.Context, dollar_1 uuid.UUID) error {
_, err := q.db.Exec(ctx, deleteAttachment, dollar_1)
return err
}
const getAttachmentByID = `-- name: GetAttachmentByID :one
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at FROM attachments
WHERE id = $1::uuid
@@ -225,3 +232,60 @@ func (q *Queries) ListMessageAttachmentsByMessageIDs(ctx context.Context, messag
}
return items, nil
}
const listStaleAttachmentUploads = `-- name: ListStaleAttachmentUploads :many
SELECT id, s3_key FROM attachments
WHERE status = 'queued' AND created_at < $1::timestamptz
`
type ListStaleAttachmentUploadsRow struct {
ID uuid.UUID
S3Key string
}
func (q *Queries) ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]ListStaleAttachmentUploadsRow, error) {
rows, err := q.db.Query(ctx, listStaleAttachmentUploads, cutoff)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListStaleAttachmentUploadsRow{}
for rows.Next() {
var i ListStaleAttachmentUploadsRow
if err := rows.Scan(&i.ID, &i.S3Key); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const markAttachmentReady = `-- name: MarkAttachmentReady :one
UPDATE attachments
SET status = 'ready', updated_at = now()
WHERE id = $1::uuid
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at
`
func (q *Queries) MarkAttachmentReady(ctx context.Context, dollar_1 uuid.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, markAttachmentReady, dollar_1)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.S3Key,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
+3 -21
View File
@@ -24,7 +24,6 @@ var (
ErrObjectNotFound = errors.New("object not found")
ErrPutObject = errors.New("put object")
ErrGetObject = errors.New("get object")
ErrHeadObject = errors.New("head object")
ErrDeleteObject = errors.New("delete object")
)
@@ -145,25 +144,8 @@ func (c *Client) Get(ctx context.Context, key string) (*awss3.GetObjectOutput, e
return out, nil
}
func (c *Client) Head(ctx context.Context, key string) (*awss3.HeadObjectOutput, error) {
out, err := c.api.HeadObject(ctx, &awss3.HeadObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(key),
})
if isNotFound(err) {
c.log.Warn("head object", "bucket", c.bucket, "key", key, "error", err)
return nil, ErrObjectNotFound
}
if err != nil {
c.log.Error("head object", "bucket", c.bucket, "key", key, "error", err)
return nil, ErrHeadObject
}
return out, nil
}
func (c *Client) Delete(ctx context.Context, key string) error {
_, err := c.api.DeleteObject(ctx, &awss3.DeleteObjectInput{
func (c *Client) DeleteAdmin(ctx context.Context, key string) error {
_, err := c.bootstrap.DeleteObject(ctx, &awss3.DeleteObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(key),
})
@@ -171,7 +153,7 @@ func (c *Client) Delete(ctx context.Context, key string) error {
return nil
}
if err != nil {
c.log.Error("delete object", "bucket", c.bucket, "key", key, "error", err)
c.log.Error("delete admin object", "bucket", c.bucket, "key", key, "error", err)
return ErrDeleteObject
}
+14 -5
View File
@@ -30,6 +30,7 @@ type attachmentQueries interface {
CreateAttachment(ctx context.Context, arg db.CreateAttachmentParams) (db.Attachment, error)
GetAttachmentByID(ctx context.Context, id uuid.UUID) (db.Attachment, error)
IsChatMember(ctx context.Context, arg db.IsChatMemberParams) (bool, error)
MarkAttachmentReady(ctx context.Context, id uuid.UUID) (db.Attachment, error)
}
type AttachmentService struct {
@@ -104,11 +105,8 @@ func (s *AttachmentService) Upload(
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return model.Attachment{}, err
}
if err := s.s3.Put(ctx, key, tmp, size, string(detectedType)); err != nil {
return model.Attachment{}, err
}
row, err := s.q.CreateAttachment(ctx, db.CreateAttachmentParams{
_, err = s.q.CreateAttachment(ctx, db.CreateAttachmentParams{
ID: id,
ChatID: chatID,
OwnerID: userID,
@@ -117,9 +115,20 @@ func (s *AttachmentService) Upload(
ContentType: db.AttachmentContentType(detectedType),
SizeBytes: int(size),
Sha256: hash,
Status: db.AttachmentStatusReady,
ErrorCode: "",
})
if postgres.IsNotFound(err) {
return model.Attachment{}, ErrAttachmentNotFound
}
if err != nil {
return model.Attachment{}, err
}
if err := s.s3.Put(ctx, key, tmp, size, string(detectedType)); err != nil {
return model.Attachment{}, err
}
row, err := s.q.MarkAttachmentReady(ctx, id)
if err != nil {
return model.Attachment{}, err
}