Проверяйте ссылки на объекты хранилища по страницам

This commit is contained in:
2026-11-20 10:54:34 +03:00
parent bdc072aab7
commit 7898074f19
2 changed files with 153 additions and 39 deletions
+84 -39
View File
@@ -144,9 +144,11 @@ func (s *reconcileRun) objects(prefix string) error {
var known map[string]bool
if prefix == "blobs/" {
known, err = s.knownFiles(page)
if err != nil {
return err
}
} else {
known, err = s.knownJobObjects(s.ctx, s.r.db, prefix, page)
}
if err != nil {
return err
}
for _, obj := range page {
if obj.Key <= after {
@@ -154,12 +156,12 @@ func (s *reconcileRun) objects(prefix string) error {
}
after = obj.Key
s.stats.ScannedObjects++
if !s.old(obj.Modified) {
if !s.old(obj.Modified) || known[obj.Key] {
continue
}
var finding *Finding
if prefix == "blobs/" {
finding, err = s.orphan(obj, known[obj.Key])
finding, err = s.orphan(obj)
} else {
finding, err = s.jobObject(prefix, obj)
}
@@ -189,38 +191,84 @@ func (s *reconcileRun) delete(key string) error {
return err
}
// Failed sources remain available for operator replay until the retention period after failure elapses.
// Staging shared by several pipeline versions is retained while any active job needs it.
func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
parts := strings.Split(strings.TrimPrefix(obj.Key, prefix), "/")
id, idErr := uuid.Parse(parts[0])
kind := "stale_staging"
query := `SELECT EXISTS(SELECT 1 FROM attachment_jobs
WHERE attachment_id=$1 AND source_key=$2 AND
(state IN ('queued','processing') OR (state='failed' AND updated_at >= now()-$3::bigint*interval '1 second')))`
args := []any{id, obj.Key, int64(s.r.retention / time.Second)}
if prefix == "variants/" {
if len(parts) != 4 || idErr != nil {
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
func parseJobObject(prefix, key string) (uuid.UUID, int32, bool) {
parts := strings.Split(strings.TrimPrefix(key, prefix), "/")
id, err := uuid.Parse(parts[0])
if prefix == "staging/" {
return id, 0, err == nil
}
if len(parts) != 4 || err != nil {
return uuid.Nil, 0, false
}
version, err := strconv.ParseInt(parts[1], 10, 32)
return id, int32(version), err == nil && version > 0
}
type jobObjectReader interface {
Query(context.Context, string, ...any) (pgx.Rows, error)
}
// The same lookup filters an S3 page and rechecks a single candidate inside its deletion transaction.
func (s *reconcileRun) knownJobObjects(ctx context.Context, db jobObjectReader, prefix string, page []Object) (map[string]bool, error) {
var ids []uuid.UUID
var keys []string
var versions []int32
for _, obj := range page {
if !s.old(obj.Modified) {
continue
}
version, err := strconv.Atoi(parts[1])
if err != nil || version < 1 || version > math.MaxInt32 {
id, version, ok := parseJobObject(prefix, obj.Key)
if !ok {
continue
}
ids = append(ids, id)
keys = append(keys, obj.Key)
if prefix == "variants/" {
versions = append(versions, version)
}
}
if len(keys) == 0 {
return nil, nil
}
var query string
var args []any
if prefix == "variants/" {
query = `SELECT o.key FROM unnest($1::uuid[], $2::text[], $3::integer[]) AS o(id,key,version)
WHERE EXISTS (SELECT 1 FROM attachment_variants v
WHERE v.attachment_id=o.id AND v.pipeline_version=o.version AND v.object_key=o.key)
OR EXISTS (SELECT 1 FROM attachment_jobs j
WHERE j.attachment_id=o.id AND j.pipeline_version=o.version AND j.state IN ('queued','processing'))`
args = []any{ids, keys, versions}
} else {
query = `SELECT o.key FROM unnest($1::uuid[], $2::text[]) AS o(id,key)
WHERE EXISTS (SELECT 1 FROM attachment_jobs j WHERE j.attachment_id=o.id AND j.source_key=o.key AND
(j.state IN ('queued','processing') OR (j.state='failed' AND j.updated_at >= now()-$3::bigint*interval '1 second')))`
args = []any{ids, keys, int64(s.r.retention / time.Second)}
}
s.stats.SQLQueries++
rows, err := db.Query(ctx, query, args...)
if err != nil {
return nil, err
}
known := make(map[string]bool)
var key string
_, err = pgx.ForEachRow(rows, []any{&key}, func() error {
known[key] = true
return nil
})
return known, err
}
func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
id, _, valid := parseJobObject(prefix, obj.Key)
kind := "stale_staging"
if prefix == "variants/" {
if !valid {
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
}
kind = "orphan_variant"
query = `SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$3 AND object_key=$2)
OR EXISTS(SELECT 1 FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=$3 AND state IN ('queued','processing'))`
args[2] = version
}
finding := &Finding{Kind: kind, Key: obj.Key, Action: "delete_object"}
var keep bool
s.stats.SQLQueries++
if err := s.r.db.QueryRow(s.ctx, query, args...).Scan(&keep); err != nil {
return nil, err
}
if keep {
return nil, nil
}
if !s.apply {
return finding, nil
}
@@ -228,17 +276,17 @@ func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
return s.locked(keyHash[:], func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
// Intake locks the staging key before inserting an attachment. Replay,
// reprocess and completion lock the existing attachment before changing job state.
if idErr == nil {
if valid {
s.stats.SQLQueries++
if _, err := tx.Exec(opCtx, `SELECT id FROM attachments WHERE id=$1 FOR UPDATE`, id); err != nil {
return nil, err
}
}
s.stats.SQLQueries++
if err := tx.QueryRow(opCtx, query, args...).Scan(&keep); err != nil {
known, err := s.knownJobObjects(opCtx, tx, prefix, []Object{obj})
if err != nil {
return nil, err
}
if keep {
if known[obj.Key] {
return nil, nil
}
op := *s
@@ -293,14 +341,11 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
return known, rows.Err()
}
func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) {
func (s *reconcileRun) orphan(obj Object) (*Finding, error) {
hash, err := hex.DecodeString(obj.Key[len("blobs/"):])
if err != nil || len(hash) != sha256.Size || BlobKey(hash) != obj.Key {
return &Finding{Kind: "invalid_blob_key", Key: obj.Key, Action: "report"}, nil
}
if known {
return nil, nil
}
f := &Finding{Kind: "orphan_blob", Key: obj.Key, Action: "delete_object"}
if !s.apply {
return f, nil
+69
View File
@@ -499,6 +499,75 @@ func TestReconciler(t *testing.T) {
t.Fatalf("per-object SQL lookup: %+v %v", stats, err)
}
})
t.Run("published variants and active staging sources are checked by page", func(t *testing.T) {
f := reconcileSetup(t, db)
const count = 100
old := time.Now().Add(-2 * time.Hour)
digest := strings.Repeat("0", 64)
hash := f.file("shared original", "ready", 2*time.Hour, false, true)
ids := make([]uuid.UUID, count)
for i := range ids {
id := uuid.New()
ids[i] = id
f.b.add("staging/"+id.String(), []byte("x"), old)
for _, name := range []string{"thumbnail", "crop", "optimized"} {
key := fmt.Sprintf("variants/%s/1/%s/%s", id, name, digest)
f.b.add(key, []byte("x"), old)
}
}
f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at,target_version,published_version)
SELECT id,$2,$3,'batch.png',$4,now(),2,1 FROM unnest($1::uuid[]) id`, ids, f.chat, f.user, hash)
f.exec(`INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key,state)
SELECT id,1,'staging/'||id,'ready' FROM attachments UNION ALL
SELECT id,2,'staging/'||id,'queued' FROM attachments`)
f.exec(`INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256)
SELECT id,1,name,'variants/'||id||'/1/'||name||'/'||$1,
CASE WHEN name='optimized' THEN 'image/webp' ELSE 'image/jpeg' END,1,1,1,decode($1,'hex')
FROM attachments CROSS JOIN unnest(ARRAY['thumbnail','crop','optimized']) name`, digest)
stats, _, err := f.run(true, false)
t.Logf("%d published variants and %d active staging sources: %+v", 3*count, count, stats)
if err != nil || stats.Findings != 0 || stats.ScannedObjects != 4*count+1 || stats.SQLQueries > 6 {
t.Fatalf("expected page-batched lookups to preserve all referenced objects: %+v, %v", stats, err)
}
})
t.Run("new job protects a variant after the page lookup", func(t *testing.T) {
f := reconcileSetup(t, db)
hash := f.file("pending source", "pending", 2*time.Hour, true, false)
var id uuid.UUID
if err := f.db.QueryRow(ctx, `SELECT id FROM attachments WHERE file_hash=$1`, hash).Scan(&id); err != nil {
t.Fatal(err)
}
key := fmt.Sprintf("variants/%s/1/thumbnail/%064x", id, 0)
f.b.add(key, []byte("x"), time.Now().Add(-2*time.Hour))
tx, err := f.db.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(ctx)
keyHash := sha256.Sum256([]byte(key))
if err := attachment.LockHash(ctx, tx, keyHash[:]); err != nil {
t.Fatal(err)
}
r, err := attachment.NewReconciler(f.db, f.b, 0)
if err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
go func() {
_, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
done <- err
}()
waitReconcileLock(t, ctx, f.db)
if _, err := tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id, source_key) VALUES ($1,$2)`, id, "staging/"+id.String()); err != nil {
t.Fatal(err)
}
if err := tx.Commit(ctx); err != nil {
t.Fatal(err)
}
if err := <-done; err != nil || !f.b.has(key) {
t.Fatalf("variant required by new job was lost: %v", err)
}
})
}
func waitReconcileLock(t *testing.T, ctx context.Context, db *pgxpool.Pool) {