diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go index a2500b47..0d78dec7 100644 --- a/internal/attachment/reconcile.go +++ b/internal/attachment/reconcile.go @@ -144,9 +144,11 @@ func (s *reconcileRun) objects(prefix string) error { var known map[string]bool if prefix == "blobs/" { known, err = s.knownFiles(page) - if err != nil { - return err - } + } else { + known, err = s.knownJobObjects(s.ctx, s.r.db, prefix, page) + } + if err != nil { + return err } for _, obj := range page { if obj.Key <= after { @@ -154,12 +156,12 @@ func (s *reconcileRun) objects(prefix string) error { } after = obj.Key s.stats.ScannedObjects++ - if !s.old(obj.Modified) { + if !s.old(obj.Modified) || known[obj.Key] { continue } var finding *Finding if prefix == "blobs/" { - finding, err = s.orphan(obj, known[obj.Key]) + finding, err = s.orphan(obj) } else { finding, err = s.jobObject(prefix, obj) } @@ -189,38 +191,84 @@ func (s *reconcileRun) delete(key string) error { return err } -// Failed sources remain available for operator replay until the retention period after failure elapses. -// Staging shared by several pipeline versions is retained while any active job needs it. -func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) { - parts := strings.Split(strings.TrimPrefix(obj.Key, prefix), "/") - id, idErr := uuid.Parse(parts[0]) - kind := "stale_staging" - query := `SELECT EXISTS(SELECT 1 FROM attachment_jobs - WHERE attachment_id=$1 AND source_key=$2 AND - (state IN ('queued','processing') OR (state='failed' AND updated_at >= now()-$3::bigint*interval '1 second')))` - args := []any{id, obj.Key, int64(s.r.retention / time.Second)} - if prefix == "variants/" { - if len(parts) != 4 || idErr != nil { - return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil +func parseJobObject(prefix, key string) (uuid.UUID, int32, bool) { + parts := strings.Split(strings.TrimPrefix(key, prefix), "/") + id, err := uuid.Parse(parts[0]) + if prefix == "staging/" { + return id, 0, err == nil + } + if len(parts) != 4 || err != nil { + return uuid.Nil, 0, false + } + version, err := strconv.ParseInt(parts[1], 10, 32) + return id, int32(version), err == nil && version > 0 +} + +type jobObjectReader interface { + Query(context.Context, string, ...any) (pgx.Rows, error) +} + +// The same lookup filters an S3 page and rechecks a single candidate inside its deletion transaction. +func (s *reconcileRun) knownJobObjects(ctx context.Context, db jobObjectReader, prefix string, page []Object) (map[string]bool, error) { + var ids []uuid.UUID + var keys []string + var versions []int32 + for _, obj := range page { + if !s.old(obj.Modified) { + continue } - version, err := strconv.Atoi(parts[1]) - if err != nil || version < 1 || version > math.MaxInt32 { + id, version, ok := parseJobObject(prefix, obj.Key) + if !ok { + continue + } + ids = append(ids, id) + keys = append(keys, obj.Key) + if prefix == "variants/" { + versions = append(versions, version) + } + } + if len(keys) == 0 { + return nil, nil + } + var query string + var args []any + if prefix == "variants/" { + query = `SELECT o.key FROM unnest($1::uuid[], $2::text[], $3::integer[]) AS o(id,key,version) + WHERE EXISTS (SELECT 1 FROM attachment_variants v + WHERE v.attachment_id=o.id AND v.pipeline_version=o.version AND v.object_key=o.key) + OR EXISTS (SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id=o.id AND j.pipeline_version=o.version AND j.state IN ('queued','processing'))` + args = []any{ids, keys, versions} + } else { + query = `SELECT o.key FROM unnest($1::uuid[], $2::text[]) AS o(id,key) + WHERE EXISTS (SELECT 1 FROM attachment_jobs j WHERE j.attachment_id=o.id AND j.source_key=o.key AND + (j.state IN ('queued','processing') OR (j.state='failed' AND j.updated_at >= now()-$3::bigint*interval '1 second')))` + args = []any{ids, keys, int64(s.r.retention / time.Second)} + } + s.stats.SQLQueries++ + rows, err := db.Query(ctx, query, args...) + if err != nil { + return nil, err + } + known := make(map[string]bool) + var key string + _, err = pgx.ForEachRow(rows, []any{&key}, func() error { + known[key] = true + return nil + }) + return known, err +} + +func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) { + id, _, valid := parseJobObject(prefix, obj.Key) + kind := "stale_staging" + if prefix == "variants/" { + if !valid { return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil } kind = "orphan_variant" - query = `SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$3 AND object_key=$2) - OR EXISTS(SELECT 1 FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=$3 AND state IN ('queued','processing'))` - args[2] = version } finding := &Finding{Kind: kind, Key: obj.Key, Action: "delete_object"} - var keep bool - s.stats.SQLQueries++ - if err := s.r.db.QueryRow(s.ctx, query, args...).Scan(&keep); err != nil { - return nil, err - } - if keep { - return nil, nil - } if !s.apply { return finding, nil } @@ -228,17 +276,17 @@ func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) { return s.locked(keyHash[:], func(opCtx context.Context, tx pgx.Tx) (*Finding, error) { // Intake locks the staging key before inserting an attachment. Replay, // reprocess and completion lock the existing attachment before changing job state. - if idErr == nil { + if valid { s.stats.SQLQueries++ if _, err := tx.Exec(opCtx, `SELECT id FROM attachments WHERE id=$1 FOR UPDATE`, id); err != nil { return nil, err } } - s.stats.SQLQueries++ - if err := tx.QueryRow(opCtx, query, args...).Scan(&keep); err != nil { + known, err := s.knownJobObjects(opCtx, tx, prefix, []Object{obj}) + if err != nil { return nil, err } - if keep { + if known[obj.Key] { return nil, nil } op := *s @@ -293,14 +341,11 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) { return known, rows.Err() } -func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) { +func (s *reconcileRun) orphan(obj Object) (*Finding, error) { hash, err := hex.DecodeString(obj.Key[len("blobs/"):]) if err != nil || len(hash) != sha256.Size || BlobKey(hash) != obj.Key { return &Finding{Kind: "invalid_blob_key", Key: obj.Key, Action: "report"}, nil } - if known { - return nil, nil - } f := &Finding{Kind: "orphan_blob", Key: obj.Key, Action: "delete_object"} if !s.apply { return f, nil diff --git a/internal/attachment/reconcile_test.go b/internal/attachment/reconcile_test.go index be689dc4..674aed57 100644 --- a/internal/attachment/reconcile_test.go +++ b/internal/attachment/reconcile_test.go @@ -499,6 +499,75 @@ func TestReconciler(t *testing.T) { t.Fatalf("per-object SQL lookup: %+v %v", stats, err) } }) + t.Run("published variants and active staging sources are checked by page", func(t *testing.T) { + f := reconcileSetup(t, db) + const count = 100 + old := time.Now().Add(-2 * time.Hour) + digest := strings.Repeat("0", 64) + hash := f.file("shared original", "ready", 2*time.Hour, false, true) + ids := make([]uuid.UUID, count) + for i := range ids { + id := uuid.New() + ids[i] = id + f.b.add("staging/"+id.String(), []byte("x"), old) + for _, name := range []string{"thumbnail", "crop", "optimized"} { + key := fmt.Sprintf("variants/%s/1/%s/%s", id, name, digest) + f.b.add(key, []byte("x"), old) + } + } + f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at,target_version,published_version) + SELECT id,$2,$3,'batch.png',$4,now(),2,1 FROM unnest($1::uuid[]) id`, ids, f.chat, f.user, hash) + f.exec(`INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key,state) + SELECT id,1,'staging/'||id,'ready' FROM attachments UNION ALL + SELECT id,2,'staging/'||id,'queued' FROM attachments`) + f.exec(`INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256) + SELECT id,1,name,'variants/'||id||'/1/'||name||'/'||$1, + CASE WHEN name='optimized' THEN 'image/webp' ELSE 'image/jpeg' END,1,1,1,decode($1,'hex') + FROM attachments CROSS JOIN unnest(ARRAY['thumbnail','crop','optimized']) name`, digest) + stats, _, err := f.run(true, false) + t.Logf("%d published variants and %d active staging sources: %+v", 3*count, count, stats) + if err != nil || stats.Findings != 0 || stats.ScannedObjects != 4*count+1 || stats.SQLQueries > 6 { + t.Fatalf("expected page-batched lookups to preserve all referenced objects: %+v, %v", stats, err) + } + }) + t.Run("new job protects a variant after the page lookup", func(t *testing.T) { + f := reconcileSetup(t, db) + hash := f.file("pending source", "pending", 2*time.Hour, true, false) + var id uuid.UUID + if err := f.db.QueryRow(ctx, `SELECT id FROM attachments WHERE file_hash=$1`, hash).Scan(&id); err != nil { + t.Fatal(err) + } + key := fmt.Sprintf("variants/%s/1/thumbnail/%064x", id, 0) + f.b.add(key, []byte("x"), time.Now().Add(-2*time.Hour)) + tx, err := f.db.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + keyHash := sha256.Sum256([]byte(key)) + if err := attachment.LockHash(ctx, tx, keyHash[:]); err != nil { + t.Fatal(err) + } + r, err := attachment.NewReconciler(f.db, f.b, 0) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil) + done <- err + }() + waitReconcileLock(t, ctx, f.db) + if _, err := tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id, source_key) VALUES ($1,$2)`, id, "staging/"+id.String()); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil || !f.b.has(key) { + t.Fatalf("variant required by new job was lost: %v", err) + } + }) } func waitReconcileLock(t *testing.T, ctx context.Context, db *pgxpool.Pool) {