Проверяйте ссылки на объекты хранилища по страницам
This commit is contained in:
@@ -144,9 +144,11 @@ func (s *reconcileRun) objects(prefix string) error {
|
||||
var known map[string]bool
|
||||
if prefix == "blobs/" {
|
||||
known, err = s.knownFiles(page)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
known, err = s.knownJobObjects(s.ctx, s.r.db, prefix, page)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, obj := range page {
|
||||
if obj.Key <= after {
|
||||
@@ -154,12 +156,12 @@ func (s *reconcileRun) objects(prefix string) error {
|
||||
}
|
||||
after = obj.Key
|
||||
s.stats.ScannedObjects++
|
||||
if !s.old(obj.Modified) {
|
||||
if !s.old(obj.Modified) || known[obj.Key] {
|
||||
continue
|
||||
}
|
||||
var finding *Finding
|
||||
if prefix == "blobs/" {
|
||||
finding, err = s.orphan(obj, known[obj.Key])
|
||||
finding, err = s.orphan(obj)
|
||||
} else {
|
||||
finding, err = s.jobObject(prefix, obj)
|
||||
}
|
||||
@@ -189,38 +191,84 @@ func (s *reconcileRun) delete(key string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Failed sources remain available for operator replay until the retention period after failure elapses.
|
||||
// Staging shared by several pipeline versions is retained while any active job needs it.
|
||||
func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
|
||||
parts := strings.Split(strings.TrimPrefix(obj.Key, prefix), "/")
|
||||
id, idErr := uuid.Parse(parts[0])
|
||||
kind := "stale_staging"
|
||||
query := `SELECT EXISTS(SELECT 1 FROM attachment_jobs
|
||||
WHERE attachment_id=$1 AND source_key=$2 AND
|
||||
(state IN ('queued','processing') OR (state='failed' AND updated_at >= now()-$3::bigint*interval '1 second')))`
|
||||
args := []any{id, obj.Key, int64(s.r.retention / time.Second)}
|
||||
if prefix == "variants/" {
|
||||
if len(parts) != 4 || idErr != nil {
|
||||
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
|
||||
func parseJobObject(prefix, key string) (uuid.UUID, int32, bool) {
|
||||
parts := strings.Split(strings.TrimPrefix(key, prefix), "/")
|
||||
id, err := uuid.Parse(parts[0])
|
||||
if prefix == "staging/" {
|
||||
return id, 0, err == nil
|
||||
}
|
||||
if len(parts) != 4 || err != nil {
|
||||
return uuid.Nil, 0, false
|
||||
}
|
||||
version, err := strconv.ParseInt(parts[1], 10, 32)
|
||||
return id, int32(version), err == nil && version > 0
|
||||
}
|
||||
|
||||
type jobObjectReader interface {
|
||||
Query(context.Context, string, ...any) (pgx.Rows, error)
|
||||
}
|
||||
|
||||
// The same lookup filters an S3 page and rechecks a single candidate inside its deletion transaction.
|
||||
func (s *reconcileRun) knownJobObjects(ctx context.Context, db jobObjectReader, prefix string, page []Object) (map[string]bool, error) {
|
||||
var ids []uuid.UUID
|
||||
var keys []string
|
||||
var versions []int32
|
||||
for _, obj := range page {
|
||||
if !s.old(obj.Modified) {
|
||||
continue
|
||||
}
|
||||
version, err := strconv.Atoi(parts[1])
|
||||
if err != nil || version < 1 || version > math.MaxInt32 {
|
||||
id, version, ok := parseJobObject(prefix, obj.Key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, id)
|
||||
keys = append(keys, obj.Key)
|
||||
if prefix == "variants/" {
|
||||
versions = append(versions, version)
|
||||
}
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var query string
|
||||
var args []any
|
||||
if prefix == "variants/" {
|
||||
query = `SELECT o.key FROM unnest($1::uuid[], $2::text[], $3::integer[]) AS o(id,key,version)
|
||||
WHERE EXISTS (SELECT 1 FROM attachment_variants v
|
||||
WHERE v.attachment_id=o.id AND v.pipeline_version=o.version AND v.object_key=o.key)
|
||||
OR EXISTS (SELECT 1 FROM attachment_jobs j
|
||||
WHERE j.attachment_id=o.id AND j.pipeline_version=o.version AND j.state IN ('queued','processing'))`
|
||||
args = []any{ids, keys, versions}
|
||||
} else {
|
||||
query = `SELECT o.key FROM unnest($1::uuid[], $2::text[]) AS o(id,key)
|
||||
WHERE EXISTS (SELECT 1 FROM attachment_jobs j WHERE j.attachment_id=o.id AND j.source_key=o.key AND
|
||||
(j.state IN ('queued','processing') OR (j.state='failed' AND j.updated_at >= now()-$3::bigint*interval '1 second')))`
|
||||
args = []any{ids, keys, int64(s.r.retention / time.Second)}
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
rows, err := db.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
known := make(map[string]bool)
|
||||
var key string
|
||||
_, err = pgx.ForEachRow(rows, []any{&key}, func() error {
|
||||
known[key] = true
|
||||
return nil
|
||||
})
|
||||
return known, err
|
||||
}
|
||||
|
||||
func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
|
||||
id, _, valid := parseJobObject(prefix, obj.Key)
|
||||
kind := "stale_staging"
|
||||
if prefix == "variants/" {
|
||||
if !valid {
|
||||
return &Finding{Kind: "invalid_variant_key", Key: obj.Key, Action: "report"}, nil
|
||||
}
|
||||
kind = "orphan_variant"
|
||||
query = `SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$3 AND object_key=$2)
|
||||
OR EXISTS(SELECT 1 FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=$3 AND state IN ('queued','processing'))`
|
||||
args[2] = version
|
||||
}
|
||||
finding := &Finding{Kind: kind, Key: obj.Key, Action: "delete_object"}
|
||||
var keep bool
|
||||
s.stats.SQLQueries++
|
||||
if err := s.r.db.QueryRow(s.ctx, query, args...).Scan(&keep); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if keep {
|
||||
return nil, nil
|
||||
}
|
||||
if !s.apply {
|
||||
return finding, nil
|
||||
}
|
||||
@@ -228,17 +276,17 @@ func (s *reconcileRun) jobObject(prefix string, obj Object) (*Finding, error) {
|
||||
return s.locked(keyHash[:], func(opCtx context.Context, tx pgx.Tx) (*Finding, error) {
|
||||
// Intake locks the staging key before inserting an attachment. Replay,
|
||||
// reprocess and completion lock the existing attachment before changing job state.
|
||||
if idErr == nil {
|
||||
if valid {
|
||||
s.stats.SQLQueries++
|
||||
if _, err := tx.Exec(opCtx, `SELECT id FROM attachments WHERE id=$1 FOR UPDATE`, id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
s.stats.SQLQueries++
|
||||
if err := tx.QueryRow(opCtx, query, args...).Scan(&keep); err != nil {
|
||||
known, err := s.knownJobObjects(opCtx, tx, prefix, []Object{obj})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if keep {
|
||||
if known[obj.Key] {
|
||||
return nil, nil
|
||||
}
|
||||
op := *s
|
||||
@@ -293,14 +341,11 @@ func (s *reconcileRun) knownFiles(page []Object) (map[string]bool, error) {
|
||||
return known, rows.Err()
|
||||
}
|
||||
|
||||
func (s *reconcileRun) orphan(obj Object, known bool) (*Finding, error) {
|
||||
func (s *reconcileRun) orphan(obj Object) (*Finding, error) {
|
||||
hash, err := hex.DecodeString(obj.Key[len("blobs/"):])
|
||||
if err != nil || len(hash) != sha256.Size || BlobKey(hash) != obj.Key {
|
||||
return &Finding{Kind: "invalid_blob_key", Key: obj.Key, Action: "report"}, nil
|
||||
}
|
||||
if known {
|
||||
return nil, nil
|
||||
}
|
||||
f := &Finding{Kind: "orphan_blob", Key: obj.Key, Action: "delete_object"}
|
||||
if !s.apply {
|
||||
return f, nil
|
||||
|
||||
@@ -499,6 +499,75 @@ func TestReconciler(t *testing.T) {
|
||||
t.Fatalf("per-object SQL lookup: %+v %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("published variants and active staging sources are checked by page", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
const count = 100
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
digest := strings.Repeat("0", 64)
|
||||
hash := f.file("shared original", "ready", 2*time.Hour, false, true)
|
||||
ids := make([]uuid.UUID, count)
|
||||
for i := range ids {
|
||||
id := uuid.New()
|
||||
ids[i] = id
|
||||
f.b.add("staging/"+id.String(), []byte("x"), old)
|
||||
for _, name := range []string{"thumbnail", "crop", "optimized"} {
|
||||
key := fmt.Sprintf("variants/%s/1/%s/%s", id, name, digest)
|
||||
f.b.add(key, []byte("x"), old)
|
||||
}
|
||||
}
|
||||
f.exec(`INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at,target_version,published_version)
|
||||
SELECT id,$2,$3,'batch.png',$4,now(),2,1 FROM unnest($1::uuid[]) id`, ids, f.chat, f.user, hash)
|
||||
f.exec(`INSERT INTO attachment_jobs(attachment_id,pipeline_version,source_key,state)
|
||||
SELECT id,1,'staging/'||id,'ready' FROM attachments UNION ALL
|
||||
SELECT id,2,'staging/'||id,'queued' FROM attachments`)
|
||||
f.exec(`INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,mime_type,width,height,size_bytes,sha256)
|
||||
SELECT id,1,name,'variants/'||id||'/1/'||name||'/'||$1,
|
||||
CASE WHEN name='optimized' THEN 'image/webp' ELSE 'image/jpeg' END,1,1,1,decode($1,'hex')
|
||||
FROM attachments CROSS JOIN unnest(ARRAY['thumbnail','crop','optimized']) name`, digest)
|
||||
stats, _, err := f.run(true, false)
|
||||
t.Logf("%d published variants and %d active staging sources: %+v", 3*count, count, stats)
|
||||
if err != nil || stats.Findings != 0 || stats.ScannedObjects != 4*count+1 || stats.SQLQueries > 6 {
|
||||
t.Fatalf("expected page-batched lookups to preserve all referenced objects: %+v, %v", stats, err)
|
||||
}
|
||||
})
|
||||
t.Run("new job protects a variant after the page lookup", func(t *testing.T) {
|
||||
f := reconcileSetup(t, db)
|
||||
hash := f.file("pending source", "pending", 2*time.Hour, true, false)
|
||||
var id uuid.UUID
|
||||
if err := f.db.QueryRow(ctx, `SELECT id FROM attachments WHERE file_hash=$1`, hash).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := fmt.Sprintf("variants/%s/1/thumbnail/%064x", id, 0)
|
||||
f.b.add(key, []byte("x"), time.Now().Add(-2*time.Hour))
|
||||
tx, err := f.db.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
keyHash := sha256.Sum256([]byte(key))
|
||||
if err := attachment.LockHash(ctx, tx, keyHash[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := attachment.NewReconciler(f.db, f.b, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := r.Run(ctx, attachment.ReconcileOptions{Apply: true}, nil)
|
||||
done <- err
|
||||
}()
|
||||
waitReconcileLock(t, ctx, f.db)
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO attachment_jobs(attachment_id, source_key) VALUES ($1,$2)`, id, "staging/"+id.String()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-done; err != nil || !f.b.has(key) {
|
||||
t.Fatalf("variant required by new job was lost: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func waitReconcileLock(t *testing.T, ctx context.Context, db *pgxpool.Pool) {
|
||||
|
||||
Reference in New Issue
Block a user