diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go index 131741f3..79db14c9 100644 --- a/cmd/reconcile/main.go +++ b/cmd/reconcile/main.go @@ -59,12 +59,13 @@ func run() error { return err } - reconciler := reconcile.New(s3Client, db.New(pool), log) + reconciler := reconcile.New(pool, s3Client, db.New(pool), log) report, err := reconciler.Run(ctx, reconcile.Options{ - DryRun: *dryRun, - Grace: *grace, - MaxDeletes: *maxDeletes, + DryRun: *dryRun, + Grace: *grace, + MaxDeletes: *maxDeletes, + StagingRetention: cfg.S3.StagingRetention, }) if err != nil { return err @@ -96,7 +97,8 @@ func printReport(report reconcile.Report) { report.ListPages, ) fmt.Printf( - "found orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n", + "found staging_objects=%d orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n", + len(report.StagingObjects), len(report.OrphanObjects), len(report.MissingAttachments), len(report.CorruptObjects), @@ -112,6 +114,9 @@ func printReport(report reconcile.Report) { report.MarkedFailed, ) + for _, key := range report.StagingObjects { + fmt.Printf("staging_object %s\n", key) + } for _, key := range report.OrphanObjects { fmt.Printf("orphan_object %s\n", key) } diff --git a/db/queries/attachments.sql b/db/queries/attachments.sql index 42accee7..8a26c771 100644 --- a/db/queries/attachments.sql +++ b/db/queries/attachments.sql @@ -50,6 +50,11 @@ INSERT INTO message_attachments ( SELECT * FROM attachments WHERE id = $1::uuid; +-- name: LockAttachmentByStagingKey :one +SELECT * FROM attachments +WHERE staging_key = sqlc.arg(staging_key)::text +FOR UPDATE; + -- name: ListAttachmentsByIDs :many SELECT status FROM attachments @@ -69,7 +74,7 @@ WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[]) ORDER BY message_id, position; -- name: ListAttachmentsForReconcile :many -SELECT id, s3_key, status, size_bytes +SELECT id, s3_key, staging_key, status, size_bytes, updated_at FROM attachments; -- name: LockAttachmentJob :one @@ -90,11 +95,10 @@ ORDER BY lease_until, id LIMIT sqlc.arg(batch_size)::int FOR UPDATE SKIP LOCKED; --- name: MarkAttachmentReady :one +-- name: ClearAttachmentStagingKey :exec UPDATE attachments -SET status = 'ready', updated_at = now() -WHERE id = sqlc.arg(id)::uuid -RETURNING *; +SET staging_key = NULL +WHERE id = sqlc.arg(id)::uuid; -- name: MarkAttachmentFailedIfReady :execrows UPDATE attachments diff --git a/internal/config/config.go b/internal/config/config.go index 68dc214c..437d0db6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -76,6 +76,7 @@ type S3Config struct { BootstrapSecretKeyFile string MaxUploadBytes int MaxImagePixels int + StagingRetention time.Duration } func (c S3Config) APIAccessKey() (string, error) { @@ -138,6 +139,7 @@ func LoadS3() S3Config { BootstrapSecretKeyFile: getEnv("S3_BOOTSTRAP_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_bootstrap_secret_access_key"), MaxUploadBytes: getEnvPositiveInt("MAX_UPLOAD_BYTES", 10485760), MaxImagePixels: getEnvPositiveInt("MAX_IMAGE_PIXELS", 40000000), + StagingRetention: time.Duration(getEnvPositiveInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second, } } diff --git a/internal/db/attachments.sql.go b/internal/db/attachments.sql.go index f22409b7..9e2f8da4 100644 --- a/internal/db/attachments.sql.go +++ b/internal/db/attachments.sql.go @@ -67,6 +67,17 @@ func (q *Queries) ClaimAttachmentJob(ctx context.Context, arg ClaimAttachmentJob return i, err } +const clearAttachmentStagingKey = `-- name: ClearAttachmentStagingKey :exec +UPDATE attachments +SET staging_key = NULL +WHERE id = $1::uuid +` + +func (q *Queries) ClearAttachmentStagingKey(ctx context.Context, id uuid.UUID) error { + _, err := q.db.Exec(ctx, clearAttachmentStagingKey, id) + return err +} + const completeAttachmentJob = `-- name: CompleteAttachmentJob :execrows UPDATE attachments SET status = 'ready', @@ -320,15 +331,17 @@ func (q *Queries) ListAttachmentsByIDs(ctx context.Context, arg ListAttachmentsB } const listAttachmentsForReconcile = `-- name: ListAttachmentsForReconcile :many -SELECT id, s3_key, status, size_bytes +SELECT id, s3_key, staging_key, status, size_bytes, updated_at FROM attachments ` type ListAttachmentsForReconcileRow struct { - ID uuid.UUID - S3Key string - Status AttachmentStatus - SizeBytes int + ID uuid.UUID + S3Key string + StagingKey *string + Status AttachmentStatus + SizeBytes int + UpdatedAt time.Time } func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttachmentsForReconcileRow, error) { @@ -343,8 +356,10 @@ func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttach if err := rows.Scan( &i.ID, &i.S3Key, + &i.StagingKey, &i.Status, &i.SizeBytes, + &i.UpdatedAt, ); err != nil { return nil, err } @@ -415,6 +430,39 @@ func (q *Queries) ListMessageAttachmentsByMessageIDs(ctx context.Context, messag return items, nil } +const lockAttachmentByStagingKey = `-- name: LockAttachmentByStagingKey :one +SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments +WHERE staging_key = $1::text +FOR UPDATE +` + +func (q *Queries) LockAttachmentByStagingKey(ctx context.Context, stagingKey string) (Attachment, error) { + row := q.db.QueryRow(ctx, lockAttachmentByStagingKey, stagingKey) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.S3Key, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.ErrorCode, + &i.CreatedAt, + &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.Width, + &i.Height, + &i.LeaseUntil, + ) + return i, err +} + const lockAttachmentJob = `-- name: LockAttachmentJob :one SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments WHERE id = $1::uuid @@ -535,40 +583,6 @@ func (q *Queries) MarkAttachmentFailedIfReady(ctx context.Context, arg MarkAttac return result.RowsAffected(), nil } -const markAttachmentReady = `-- name: MarkAttachmentReady :one -UPDATE attachments -SET status = 'ready', updated_at = now() -WHERE id = $1::uuid -RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until -` - -func (q *Queries) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (Attachment, error) { - row := q.db.QueryRow(ctx, markAttachmentReady, id) - var i Attachment - err := row.Scan( - &i.ID, - &i.ChatID, - &i.OwnerID, - &i.S3Key, - &i.Filename, - &i.ContentType, - &i.SizeBytes, - &i.Sha256, - &i.Status, - &i.ErrorCode, - &i.CreatedAt, - &i.UpdatedAt, - &i.StagingKey, - &i.JobID, - &i.PipelineVersion, - &i.Attempts, - &i.Width, - &i.Height, - &i.LeaseUntil, - ) - return i, err -} - const recoverExpiredAttachmentJob = `-- name: RecoverExpiredAttachmentJob :execrows UPDATE attachments SET status = 'queued', diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index d66bb560..2ab00006 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -8,28 +8,34 @@ import ( "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" "mtusi2026-messenger/internal/db" s3store "mtusi2026-messenger/internal/s3" ) const ( - Prefix = "chats/" - DefaultGrace = time.Hour - DefaultMaxDeletes = 100 - ErrorCodeObjectMissing = "object_missing" + Prefix = "chats/" + StagingPrefix = "staging/" + DefaultStagingRetention = 24 * time.Hour + DefaultGrace = time.Hour + DefaultMaxDeletes = 100 + ErrorCodeObjectMissing = "object_missing" ) type Options struct { - DryRun bool - Grace time.Duration - MaxDeletes int + DryRun bool + Grace time.Duration + MaxDeletes int + StagingRetention time.Duration } type Report struct { ScannedObjects int `json:"scanned_objects"` ScannedRows int `json:"scanned_rows"` ListPages int `json:"list_pages"` + StagingObjects []string `json:"staging_objects"` OrphanObjects []string `json:"orphan_objects"` MissingAttachments []uuid.UUID `json:"missing_attachments"` CorruptObjects []string `json:"corrupt_objects"` @@ -47,32 +53,50 @@ type ObjectClient interface { DeleteAdmin(ctx context.Context, key string) error } +type txBeginner interface { + Begin(ctx context.Context) (pgx.Tx, error) +} + type AttachmentQueries interface { + LockAttachmentByStagingKey(ctx context.Context, stagingKey string) (db.Attachment, error) + ClearAttachmentStagingKey(ctx context.Context, id uuid.UUID) error ListAttachmentsForReconcile(ctx context.Context) ([]db.ListAttachmentsForReconcileRow, error) MarkAttachmentFailedIfReady(ctx context.Context, arg db.MarkAttachmentFailedIfReadyParams) (int64, error) } type Reconciler struct { - q AttachmentQueries - store ObjectClient - log *slog.Logger - now func() time.Time + pool txBeginner + q AttachmentQueries + withTx func(pgx.Tx) AttachmentQueries + store ObjectClient + log *slog.Logger + now func() time.Time } type plan struct { orphans []s3store.ObjectInfo + staging []s3store.ObjectInfo missing []db.ListAttachmentsForReconcileRow corrupt []s3store.ObjectInfo skippedFresh int skippedInactive int } -func New(store ObjectClient, queries AttachmentQueries, log *slog.Logger) *Reconciler { +func New( + pool *pgxpool.Pool, + store ObjectClient, + queries *db.Queries, + log *slog.Logger, +) *Reconciler { if log == nil { log = slog.Default() } return &Reconciler{ + pool: pool, + withTx: func(tx pgx.Tx) AttachmentQueries { + return queries.WithTx(tx) + }, store: store, q: queries, log: log, @@ -86,6 +110,7 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) { report := Report{ OrphanObjects: []string{}, + StagingObjects: []string{}, MissingAttachments: []uuid.UUID{}, CorruptObjects: []string{}, DryRun: opts.DryRun, @@ -97,20 +122,28 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) { return report, err } + staging, err := r.store.ListAll(ctx, StagingPrefix) + if err != nil { + return report, err + } + rows, err := r.q.ListAttachmentsForReconcile(ctx) if err != nil { return report, err } - report.ScannedObjects = len(listed.Objects) + report.ScannedObjects = len(listed.Objects) + len(staging.Objects) report.ScannedRows = len(rows) - report.ListPages = listed.Pages + report.ListPages = listed.Pages + staging.Pages - p := buildPlan(listed.Objects, rows, started, opts.Grace) + p := buildPlan(listed.Objects, staging.Objects, rows, started, opts) for _, obj := range p.orphans { report.OrphanObjects = append(report.OrphanObjects, obj.Key) } + for _, obj := range p.staging { + report.StagingObjects = append(report.StagingObjects, obj.Key) + } for _, obj := range p.corrupt { report.CorruptObjects = append(report.CorruptObjects, obj.Key) } @@ -134,6 +167,7 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) { "scanned_rows", report.ScannedRows, "list_pages", report.ListPages, "orphan_objects", len(report.OrphanObjects), + "staging_objects", len(report.StagingObjects), "missing_attachments", len(report.MissingAttachments), "corrupt_objects", len(report.CorruptObjects), "deleted_objects", report.DeletedObjects, @@ -152,14 +186,19 @@ func (o Options) withDefaults() Options { o.MaxDeletes = DefaultMaxDeletes } + if o.StagingRetention <= 0 { + o.StagingRetention = DefaultStagingRetention + } + return o } func buildPlan( objects []s3store.ObjectInfo, + staging []s3store.ObjectInfo, rows []db.ListAttachmentsForReconcileRow, now time.Time, - grace time.Duration, + opts Options, ) plan { var p plan @@ -168,7 +207,12 @@ func buildPlan( byKey[obj.Key] = obj } + byStagingKey := make(map[string]db.ListAttachmentsForReconcileRow, len(rows)) for _, row := range rows { + if row.StagingKey != nil { + byStagingKey[*row.StagingKey] = row + } + obj, exists := byKey[row.S3Key] if exists { delete(byKey, row.S3Key) @@ -193,13 +237,35 @@ func buildPlan( } for _, obj := range byKey { - if now.Sub(obj.LastModified) < grace { + if now.Sub(obj.LastModified) < opts.Grace { p.skippedFresh++ continue } p.orphans = append(p.orphans, obj) } + for _, obj := range staging { + retainedSince := obj.LastModified + if row, exists := byStagingKey[obj.Key]; exists { + if row.Status != db.AttachmentStatusReady && row.Status != db.AttachmentStatusFailed { + p.skippedInactive++ + continue + } + + retainedSince = row.UpdatedAt + } + + if now.Sub(retainedSince) < opts.StagingRetention { + p.skippedFresh++ + continue + } + + p.staging = append(p.staging, obj) + } + + sort.Slice(p.staging, func(i, j int) bool { + return p.staging[i].Key < p.staging[j].Key + }) sort.Slice(p.orphans, func(i, j int) bool { return p.orphans[i].Key < p.orphans[j].Key }) @@ -243,4 +309,15 @@ func (r *Reconciler) apply(ctx context.Context, opts Options, p plan, report *Re report.MarkedFailed += int(marked) } + + for _, obj := range p.staging { + if report.DeletedObjects >= opts.MaxDeletes { + report.Errors = append(report.Errors, "max deletes reached") + break + } + + if err := r.cleanupStagingObject(ctx, obj, opts, report); err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("cleanup staging %s: %v", obj.Key, err)) + } + } } diff --git a/internal/reconcile/staging.go b/internal/reconcile/staging.go new file mode 100644 index 00000000..63cafd29 --- /dev/null +++ b/internal/reconcile/staging.go @@ -0,0 +1,60 @@ +package reconcile + +import ( + "context" + "errors" + "time" + + "github.com/jackc/pgx/v5" + + "mtusi2026-messenger/internal/db" + s3store "mtusi2026-messenger/internal/s3" +) + +func (r *Reconciler) cleanupStagingObject( + ctx context.Context, + obj s3store.ObjectInfo, + opts Options, + report *Report, +) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + tx, err := r.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + + q := r.withTx(tx) + row, err := q.LockAttachmentByStagingKey(ctx, obj.Key) + orphan := errors.Is(err, pgx.ErrNoRows) + if err != nil && !orphan { + return err + } + + if !orphan { + if row.Status != db.AttachmentStatusReady && row.Status != db.AttachmentStatusFailed { + report.SkippedInactive++ + return nil + } + + if r.now().Sub(row.UpdatedAt) < opts.StagingRetention { + report.SkippedFresh++ + return nil + } + } + + if err := r.store.DeleteAdmin(ctx, obj.Key); err != nil { + return err + } + report.DeletedObjects++ + + if !orphan { + if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil { + return err + } + } + + return tx.Commit(ctx) +} diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 9a14ad41..4fbf6fd3 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -238,6 +238,13 @@ func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error { return err } + if row.StagingKey != nil { + err := w.deleteStagingObject(ctx, *row.StagingKey) + if err != nil && ctx.Err() == nil { + w.log.Warn("delete ready attachment staging", "attachment_id", row.ID, "error", err) + } + } + w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt) return nil } @@ -391,6 +398,41 @@ func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cau return nil } +func (w *Worker) deleteStagingObject(ctx context.Context, key string) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + tx, err := w.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + + q := w.q.WithTx(tx) + row, err := q.LockAttachmentByStagingKey(ctx, key) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + + if err != nil { + return err + } + + if row.Status != db.AttachmentStatusReady { + return nil + } + + if err := w.s3.DeleteAdmin(ctx, key); err != nil { + return err + } + + if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil { + return err + } + + return tx.Commit(ctx) +} + func (w *Worker) recoverLoop(ctx context.Context) { for ctx.Err() == nil { recovered, err := w.recoverOne(ctx)