feat: clean up staging objects after processing and during reconciliation

This commit is contained in:
kk10
2026-11-16 11:48:30 +03:00
parent 19152e7d18
commit b35faa5af8
7 changed files with 270 additions and 66 deletions
+10 -5
View File
@@ -59,12 +59,13 @@ func run() error {
return err
}
reconciler := reconcile.New(s3Client, db.New(pool), log)
reconciler := reconcile.New(pool, s3Client, db.New(pool), log)
report, err := reconciler.Run(ctx, reconcile.Options{
DryRun: *dryRun,
Grace: *grace,
MaxDeletes: *maxDeletes,
DryRun: *dryRun,
Grace: *grace,
MaxDeletes: *maxDeletes,
StagingRetention: cfg.S3.StagingRetention,
})
if err != nil {
return err
@@ -96,7 +97,8 @@ func printReport(report reconcile.Report) {
report.ListPages,
)
fmt.Printf(
"found orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n",
"found staging_objects=%d orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n",
len(report.StagingObjects),
len(report.OrphanObjects),
len(report.MissingAttachments),
len(report.CorruptObjects),
@@ -112,6 +114,9 @@ func printReport(report reconcile.Report) {
report.MarkedFailed,
)
for _, key := range report.StagingObjects {
fmt.Printf("staging_object %s\n", key)
}
for _, key := range report.OrphanObjects {
fmt.Printf("orphan_object %s\n", key)
}
+9 -5
View File
@@ -50,6 +50,11 @@ INSERT INTO message_attachments (
SELECT * FROM attachments
WHERE id = $1::uuid;
-- name: LockAttachmentByStagingKey :one
SELECT * FROM attachments
WHERE staging_key = sqlc.arg(staging_key)::text
FOR UPDATE;
-- name: ListAttachmentsByIDs :many
SELECT status
FROM attachments
@@ -69,7 +74,7 @@ WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
ORDER BY message_id, position;
-- name: ListAttachmentsForReconcile :many
SELECT id, s3_key, status, size_bytes
SELECT id, s3_key, staging_key, status, size_bytes, updated_at
FROM attachments;
-- name: LockAttachmentJob :one
@@ -90,11 +95,10 @@ ORDER BY lease_until, id
LIMIT sqlc.arg(batch_size)::int
FOR UPDATE SKIP LOCKED;
-- name: MarkAttachmentReady :one
-- name: ClearAttachmentStagingKey :exec
UPDATE attachments
SET status = 'ready', updated_at = now()
WHERE id = sqlc.arg(id)::uuid
RETURNING *;
SET staging_key = NULL
WHERE id = sqlc.arg(id)::uuid;
-- name: MarkAttachmentFailedIfReady :execrows
UPDATE attachments
+2
View File
@@ -76,6 +76,7 @@ type S3Config struct {
BootstrapSecretKeyFile string
MaxUploadBytes int
MaxImagePixels int
StagingRetention time.Duration
}
func (c S3Config) APIAccessKey() (string, error) {
@@ -138,6 +139,7 @@ func LoadS3() S3Config {
BootstrapSecretKeyFile: getEnv("S3_BOOTSTRAP_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_bootstrap_secret_access_key"),
MaxUploadBytes: getEnvPositiveInt("MAX_UPLOAD_BYTES", 10485760),
MaxImagePixels: getEnvPositiveInt("MAX_IMAGE_PIXELS", 40000000),
StagingRetention: time.Duration(getEnvPositiveInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second,
}
}
+53 -39
View File
@@ -67,6 +67,17 @@ func (q *Queries) ClaimAttachmentJob(ctx context.Context, arg ClaimAttachmentJob
return i, err
}
const clearAttachmentStagingKey = `-- name: ClearAttachmentStagingKey :exec
UPDATE attachments
SET staging_key = NULL
WHERE id = $1::uuid
`
func (q *Queries) ClearAttachmentStagingKey(ctx context.Context, id uuid.UUID) error {
_, err := q.db.Exec(ctx, clearAttachmentStagingKey, id)
return err
}
const completeAttachmentJob = `-- name: CompleteAttachmentJob :execrows
UPDATE attachments
SET status = 'ready',
@@ -320,15 +331,17 @@ func (q *Queries) ListAttachmentsByIDs(ctx context.Context, arg ListAttachmentsB
}
const listAttachmentsForReconcile = `-- name: ListAttachmentsForReconcile :many
SELECT id, s3_key, status, size_bytes
SELECT id, s3_key, staging_key, status, size_bytes, updated_at
FROM attachments
`
type ListAttachmentsForReconcileRow struct {
ID uuid.UUID
S3Key string
Status AttachmentStatus
SizeBytes int
ID uuid.UUID
S3Key string
StagingKey *string
Status AttachmentStatus
SizeBytes int
UpdatedAt time.Time
}
func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttachmentsForReconcileRow, error) {
@@ -343,8 +356,10 @@ func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttach
if err := rows.Scan(
&i.ID,
&i.S3Key,
&i.StagingKey,
&i.Status,
&i.SizeBytes,
&i.UpdatedAt,
); err != nil {
return nil, err
}
@@ -415,6 +430,39 @@ func (q *Queries) ListMessageAttachmentsByMessageIDs(ctx context.Context, messag
return items, nil
}
const lockAttachmentByStagingKey = `-- name: LockAttachmentByStagingKey :one
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
WHERE staging_key = $1::text
FOR UPDATE
`
func (q *Queries) LockAttachmentByStagingKey(ctx context.Context, stagingKey string) (Attachment, error) {
row := q.db.QueryRow(ctx, lockAttachmentByStagingKey, stagingKey)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.S3Key,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
&i.StagingKey,
&i.JobID,
&i.PipelineVersion,
&i.Attempts,
&i.Width,
&i.Height,
&i.LeaseUntil,
)
return i, err
}
const lockAttachmentJob = `-- name: LockAttachmentJob :one
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
WHERE id = $1::uuid
@@ -535,40 +583,6 @@ func (q *Queries) MarkAttachmentFailedIfReady(ctx context.Context, arg MarkAttac
return result.RowsAffected(), nil
}
const markAttachmentReady = `-- name: MarkAttachmentReady :one
UPDATE attachments
SET status = 'ready', updated_at = now()
WHERE id = $1::uuid
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until
`
func (q *Queries) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, markAttachmentReady, id)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.S3Key,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
&i.StagingKey,
&i.JobID,
&i.PipelineVersion,
&i.Attempts,
&i.Width,
&i.Height,
&i.LeaseUntil,
)
return i, err
}
const recoverExpiredAttachmentJob = `-- name: RecoverExpiredAttachmentJob :execrows
UPDATE attachments
SET status = 'queued',
+94 -17
View File
@@ -8,28 +8,34 @@ import (
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"mtusi2026-messenger/internal/db"
s3store "mtusi2026-messenger/internal/s3"
)
const (
Prefix = "chats/"
DefaultGrace = time.Hour
DefaultMaxDeletes = 100
ErrorCodeObjectMissing = "object_missing"
Prefix = "chats/"
StagingPrefix = "staging/"
DefaultStagingRetention = 24 * time.Hour
DefaultGrace = time.Hour
DefaultMaxDeletes = 100
ErrorCodeObjectMissing = "object_missing"
)
type Options struct {
DryRun bool
Grace time.Duration
MaxDeletes int
DryRun bool
Grace time.Duration
MaxDeletes int
StagingRetention time.Duration
}
type Report struct {
ScannedObjects int `json:"scanned_objects"`
ScannedRows int `json:"scanned_rows"`
ListPages int `json:"list_pages"`
StagingObjects []string `json:"staging_objects"`
OrphanObjects []string `json:"orphan_objects"`
MissingAttachments []uuid.UUID `json:"missing_attachments"`
CorruptObjects []string `json:"corrupt_objects"`
@@ -47,32 +53,50 @@ type ObjectClient interface {
DeleteAdmin(ctx context.Context, key string) error
}
type txBeginner interface {
Begin(ctx context.Context) (pgx.Tx, error)
}
type AttachmentQueries interface {
LockAttachmentByStagingKey(ctx context.Context, stagingKey string) (db.Attachment, error)
ClearAttachmentStagingKey(ctx context.Context, id uuid.UUID) error
ListAttachmentsForReconcile(ctx context.Context) ([]db.ListAttachmentsForReconcileRow, error)
MarkAttachmentFailedIfReady(ctx context.Context, arg db.MarkAttachmentFailedIfReadyParams) (int64, error)
}
type Reconciler struct {
q AttachmentQueries
store ObjectClient
log *slog.Logger
now func() time.Time
pool txBeginner
q AttachmentQueries
withTx func(pgx.Tx) AttachmentQueries
store ObjectClient
log *slog.Logger
now func() time.Time
}
type plan struct {
orphans []s3store.ObjectInfo
staging []s3store.ObjectInfo
missing []db.ListAttachmentsForReconcileRow
corrupt []s3store.ObjectInfo
skippedFresh int
skippedInactive int
}
func New(store ObjectClient, queries AttachmentQueries, log *slog.Logger) *Reconciler {
func New(
pool *pgxpool.Pool,
store ObjectClient,
queries *db.Queries,
log *slog.Logger,
) *Reconciler {
if log == nil {
log = slog.Default()
}
return &Reconciler{
pool: pool,
withTx: func(tx pgx.Tx) AttachmentQueries {
return queries.WithTx(tx)
},
store: store,
q: queries,
log: log,
@@ -86,6 +110,7 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
report := Report{
OrphanObjects: []string{},
StagingObjects: []string{},
MissingAttachments: []uuid.UUID{},
CorruptObjects: []string{},
DryRun: opts.DryRun,
@@ -97,20 +122,28 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
return report, err
}
staging, err := r.store.ListAll(ctx, StagingPrefix)
if err != nil {
return report, err
}
rows, err := r.q.ListAttachmentsForReconcile(ctx)
if err != nil {
return report, err
}
report.ScannedObjects = len(listed.Objects)
report.ScannedObjects = len(listed.Objects) + len(staging.Objects)
report.ScannedRows = len(rows)
report.ListPages = listed.Pages
report.ListPages = listed.Pages + staging.Pages
p := buildPlan(listed.Objects, rows, started, opts.Grace)
p := buildPlan(listed.Objects, staging.Objects, rows, started, opts)
for _, obj := range p.orphans {
report.OrphanObjects = append(report.OrphanObjects, obj.Key)
}
for _, obj := range p.staging {
report.StagingObjects = append(report.StagingObjects, obj.Key)
}
for _, obj := range p.corrupt {
report.CorruptObjects = append(report.CorruptObjects, obj.Key)
}
@@ -134,6 +167,7 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
"scanned_rows", report.ScannedRows,
"list_pages", report.ListPages,
"orphan_objects", len(report.OrphanObjects),
"staging_objects", len(report.StagingObjects),
"missing_attachments", len(report.MissingAttachments),
"corrupt_objects", len(report.CorruptObjects),
"deleted_objects", report.DeletedObjects,
@@ -152,14 +186,19 @@ func (o Options) withDefaults() Options {
o.MaxDeletes = DefaultMaxDeletes
}
if o.StagingRetention <= 0 {
o.StagingRetention = DefaultStagingRetention
}
return o
}
func buildPlan(
objects []s3store.ObjectInfo,
staging []s3store.ObjectInfo,
rows []db.ListAttachmentsForReconcileRow,
now time.Time,
grace time.Duration,
opts Options,
) plan {
var p plan
@@ -168,7 +207,12 @@ func buildPlan(
byKey[obj.Key] = obj
}
byStagingKey := make(map[string]db.ListAttachmentsForReconcileRow, len(rows))
for _, row := range rows {
if row.StagingKey != nil {
byStagingKey[*row.StagingKey] = row
}
obj, exists := byKey[row.S3Key]
if exists {
delete(byKey, row.S3Key)
@@ -193,13 +237,35 @@ func buildPlan(
}
for _, obj := range byKey {
if now.Sub(obj.LastModified) < grace {
if now.Sub(obj.LastModified) < opts.Grace {
p.skippedFresh++
continue
}
p.orphans = append(p.orphans, obj)
}
for _, obj := range staging {
retainedSince := obj.LastModified
if row, exists := byStagingKey[obj.Key]; exists {
if row.Status != db.AttachmentStatusReady && row.Status != db.AttachmentStatusFailed {
p.skippedInactive++
continue
}
retainedSince = row.UpdatedAt
}
if now.Sub(retainedSince) < opts.StagingRetention {
p.skippedFresh++
continue
}
p.staging = append(p.staging, obj)
}
sort.Slice(p.staging, func(i, j int) bool {
return p.staging[i].Key < p.staging[j].Key
})
sort.Slice(p.orphans, func(i, j int) bool {
return p.orphans[i].Key < p.orphans[j].Key
})
@@ -243,4 +309,15 @@ func (r *Reconciler) apply(ctx context.Context, opts Options, p plan, report *Re
report.MarkedFailed += int(marked)
}
for _, obj := range p.staging {
if report.DeletedObjects >= opts.MaxDeletes {
report.Errors = append(report.Errors, "max deletes reached")
break
}
if err := r.cleanupStagingObject(ctx, obj, opts, report); err != nil {
report.Errors = append(report.Errors, fmt.Sprintf("cleanup staging %s: %v", obj.Key, err))
}
}
}
+60
View File
@@ -0,0 +1,60 @@
package reconcile
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
"mtusi2026-messenger/internal/db"
s3store "mtusi2026-messenger/internal/s3"
)
func (r *Reconciler) cleanupStagingObject(
ctx context.Context,
obj s3store.ObjectInfo,
opts Options,
report *Report,
) error {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tx, err := r.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
q := r.withTx(tx)
row, err := q.LockAttachmentByStagingKey(ctx, obj.Key)
orphan := errors.Is(err, pgx.ErrNoRows)
if err != nil && !orphan {
return err
}
if !orphan {
if row.Status != db.AttachmentStatusReady && row.Status != db.AttachmentStatusFailed {
report.SkippedInactive++
return nil
}
if r.now().Sub(row.UpdatedAt) < opts.StagingRetention {
report.SkippedFresh++
return nil
}
}
if err := r.store.DeleteAdmin(ctx, obj.Key); err != nil {
return err
}
report.DeletedObjects++
if !orphan {
if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil {
return err
}
}
return tx.Commit(ctx)
}
+42
View File
@@ -238,6 +238,13 @@ func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error {
return err
}
if row.StagingKey != nil {
err := w.deleteStagingObject(ctx, *row.StagingKey)
if err != nil && ctx.Err() == nil {
w.log.Warn("delete ready attachment staging", "attachment_id", row.ID, "error", err)
}
}
w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
return nil
}
@@ -391,6 +398,41 @@ func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cau
return nil
}
func (w *Worker) deleteStagingObject(ctx context.Context, key string) error {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tx, err := w.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
q := w.q.WithTx(tx)
row, err := q.LockAttachmentByStagingKey(ctx, key)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
if row.Status != db.AttachmentStatusReady {
return nil
}
if err := w.s3.DeleteAdmin(ctx, key); err != nil {
return err
}
if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil {
return err
}
return tx.Commit(ctx)
}
func (w *Worker) recoverLoop(ctx context.Context) {
for ctx.Err() == nil {
recovered, err := w.recoverOne(ctx)